diff --git a/.gitattributes b/.gitattributes
index 73c802c3df8b2b6a540d786b1bca94b6512abedd..8b3eda465838bc582b0dcf21a10771f2791c0f0d 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -163,3 +163,132 @@ FuSe_Octo_model/50000/default/checkpoint filter=lfs diff=lfs merge=lfs -text
 15000/model_opt_state/ocdbt.process_8/d/858301ae6f38ba94adffab05b1387cba filter=lfs diff=lfs merge=lfs -text
 15000/model_opt_state/ocdbt.process_9/d/04559f6daf0bf3aaa85938d897fd2336 filter=lfs diff=lfs merge=lfs -text
 15000/model_opt_state/ocdbt.process_9/d/e6a824634176aed6f106a2d71b705cf4 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/language_tokenizer/language_tokenizer.proto filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_0/d/85bdac45edbd6c9198be15243b619a78 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_0/d/976003b263b86a7e904f54c7dcba50f4 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_1/d/26d761b04111868f51482bad86328227 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_1/d/9596eae70329fba3ab7a8617ee74d986 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_10/d/16b31b4c6b474856951719672e63b778 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_10/d/1733100935709df605fcf9878094b9c3 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_11/d/b2b557800d0ee055927d1a44b9af3eda filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_11/d/f54bc57c4228488cfdb02842f3b418b7 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_12/d/6cdcc9ab301a7f48e694d1aad2c111af filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_12/d/8291fad89e6470abe05f5d7d293a35a3 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_13/d/10c08f2bc35f069aa05ca0047138da8e filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_13/d/1d1e71a9ffa1eb758725987c74a464e3 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_14/d/a87993cacbecc6bb164093187da985c9 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_14/d/b969d167d8760c9e5a0e1d30a2cb09fd filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_15/d/0bc1e2d4fa3e10b8a5c60b904a3759f9 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_15/d/d34decd5ceac0c14e70ec8d4cbc86778 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_16/d/3677e1699d7ae709527dd2e1477141e7 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_16/d/d0371996f38251808fe6415d56841d7d filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_17/d/74374fad4fa3c3e38ebd5ef167ae059b filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_17/d/bdfd2ec86e0e010f6e71d523352a943f filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_18/d/64e689563147cf277b989a13a59c7bfc filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_18/d/a1ebcd8a00e0f80c2fbc262ab6ad00bd filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_19/d/ad4cf83a12cd3b56fac3522c2f60ad10 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_19/d/aeb56057d360535456c11e307a73d49e filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_2/d/1d38672b8446b298b569bc9ad19e2a9a filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_2/d/5fcdfb031d23ea89302996f02317f27c filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_20/d/2b6cf96aa2e47c9c97fe2ae66e007cb2 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_20/d/8d4498d6909c0b528fd53a0feaf0b3fc filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_21/d/1a7a94049854b82ccf2b4d7b58bf04bd filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_21/d/8a7f120b1cf3cbfcfba433bad6f6dc77 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_22/d/104320b48da2ee12a396f83c59100e9e filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_22/d/37afc7a232db2d2bec5277b20748e650 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_23/d/91a858e355267f36d2e4da81458a20a6 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_23/d/d528bf5395ffe5094194b1d84dab8b4e filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_24/d/17060105b41c035fcdb3de23ec5fe55b filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_24/d/eea47ac04671d41dffd1a53e4d91aafe filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_25/d/dc9640b1dfd7dc75d20d072cdef305eb filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_25/d/eeaa951f07b9a1bf11b6ae516e9ac4e0 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_26/d/40bd6d1cafcab5093472f08855ecb1fc filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_26/d/6081bb27ad0e0b66450d6ca947dc2658 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_27/d/0c599c28fa763262c933067d1094605c filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_27/d/f115e41d34a9d8fbfea8549bf8732ed1 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_28/d/5f24e1bffe456424abb16367d982a77c filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_28/d/c6527a1b344a45832ecb7038e71abcc9 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_29/d/79777c9a2c5a16d7704f8dc5d09b42bf filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_29/d/7ac60bf8900828cfd243e9fac33458ec filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_3/d/b2c13cbf2a5a73b7e86c4335f037442b filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_3/d/eb2fdd93d5376fa3952d28bc18e2379e filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_30/d/5739f93aa1ce52eb1f1a3d1c12ba4d4d filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_30/d/91f3fc48630d56e020a7a70cd78841fd filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_31/d/71162278399c9d898e1421e629bf9905 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_31/d/a496131e03c3898cb7aca70bb9d40d72 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_4/d/0e2951fa0993cfe957d501beb89ae0e1 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_4/d/b062f37880666b57ebdee2082f88e2c3 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_5/d/aae30a0c304b761b9053580bf34b386f filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_5/d/e60bd10ff8f697eab590d3d1537bde56 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_6/d/7e3314a9dc17979d12616c71052d3e8b filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_6/d/d32f8c4ba3139b8a6a3c5ec0d4be4039 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_7/d/6ae86240fa7e988b870ff2b4bab25afd filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_7/d/ddda27fafa957b578ae376ea148fa768 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_8/d/b462c404ca75a6a02e7e4fd2473c23bf filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_8/d/db63f820fe3ce3a3e4e749b70fffa79d filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_9/d/7c372b8786cf1c28dc59a79dcf2f7151 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_9/d/87a425a6d7a784b99018f979e4d422f9 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_0/d/b2b2d4192f8d1fb2721b4379ac131bb0 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_0/d/de842017dc94800c82512dc4d2c1a1cc filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_1/d/1c5a38f90f10f7fcef4697c958d1df0f filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_1/d/df49db5b82884be4ae0db6fde1a70509 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_10/d/169303b9063568ebc21d7c44c6b68487 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_10/d/2f8fa5c13c4935a6f5dcee803db11d27 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_11/d/41fd39581f2fdbb7c7f88967ad30b88f filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_11/d/e77cbf5e2beeb8eda67bc2dd4c9bd050 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_12/d/a0c1df1ccbf3948ec20816930f5242a2 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_12/d/f901b15c02b6030ae3aa85e2c14c9879 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_13/d/e5fa87781576945c2c9a6c1273f8b1b8 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_13/d/f60ec11c15c5df1107d7996760e221a5 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_14/d/2c344f840a241a8b8fadec56eed49204 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_14/d/3c62b1eef6e37ee7dcd23f28eaa55f27 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_15/d/4287aca8e4da1e46359ce16ad7537941 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_15/d/d9691fde608ced02a7180e4f75fb1e47 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_16/d/0fcea0b63e2a564a7b97a319d25a9b7f filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_16/d/4c7521528de273013c1e222445927a7a filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_17/d/3dca904f7fa08e662b395cbf58a1c0b4 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_17/d/6f0938986ed04df2c37c113af1b7a470 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_18/d/1499f8858662865a9ecc8dc296d1d189 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_18/d/cfddcc8fdccb06464e380b341619ea23 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_19/d/1e2b73aeb58359924fc9486156fea0e3 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_19/d/f6287fa59d763f6bf5dd981251eb5caa filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_2/d/3083e52ee48a8d93e95a3603946ebf2e filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_2/d/98be3aabfc8519b6a51b69134d263876 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_20/d/3186551c62f14827729fbce166596418 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_20/d/9aecae8ebf718fd8dce25c4cd24a2cdb filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_21/d/0514c5a3422c9a87562309b692e67ca8 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_21/d/27752287f91c2645c49b80c26b73e4fc filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_22/d/0b2c316b3fc5dd112bef35f7a9f91c93 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_22/d/5f6eac589f8368de171498b46e53ec98 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_23/d/a5b715ca6f77998a463f8ba8aebfbe76 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_23/d/b42eafcfa226c95a8e567ce0ef95ac98 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_24/d/6c3ccc76533c19087492265b1a738415 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_24/d/ba39e146acfe623519c669f38854e58f filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_25/d/1430470b859a65a9029b1e14a7c09dc4 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_25/d/6a64d36f4a3d51c5e190716ac9e28c71 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_26/d/10baf404782bca68d121552866e1eb70 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_26/d/2af55ac925c869dc84dfa101faf79526 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_27/d/2830f658bc973c353bbf25ab21535be6 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_27/d/dc136ca01b5b69effd00c82f7b67a732 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_28/d/51fb3b7591a83c0cae47674a1562662c filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_28/d/cb5b6764d8410576a6bcffbb43aee7c3 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_29/d/7b18440f014da88801f09c4051f683ff filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_29/d/b4cf19f1739cee0917d904b114e0c4d0 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_3/d/6584ee49a934b4100bf64dd34b963896 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_3/d/e0e8cb5ea3b67de7ad9d977f0e788dbb filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_30/d/7a53e2fb9d2ff318ecb3ecb2e9cdd3d0 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_30/d/c89415f376323bbbb2d6e57e49d819d7 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_31/d/67e947717027c4201b83f1c312a3dfd2 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_31/d/c85296320cd7fdf6c375bdca6d3e2959 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_4/d/1bae1a66cc02f46a28f25aed3c72f408 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_4/d/2d40d34f5260ec4069b288b2a0c514a0 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_5/d/df547fbbcb53d9b85fc7dfa1d7414930 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_5/d/e7269d185aa10563d65edf4cad96d800 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_6/d/33c9c913825f7d2a61f0739d28514b32 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_6/d/63c9faf5a9d0c5c9998e3240a363ee0a filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_7/d/32843871a03f7e535f0d732c2c7b4752 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_7/d/38bf128cb85362140971dc8eccff77e8 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_8/d/45852443b4b7194b9e1f571c6bd8f1dd filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_8/d/858301ae6f38ba94adffab05b1387cba filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_9/d/04559f6daf0bf3aaa85938d897fd2336 filter=lfs diff=lfs merge=lfs -text
+FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_9/d/e6a824634176aed6f106a2d71b705cf4 filter=lfs diff=lfs merge=lfs -text
diff --git a/FuSe_PaliVLA_model/15000/_CHECKPOINT_METADATA b/FuSe_PaliVLA_model/15000/_CHECKPOINT_METADATA
new file mode 100644
index 0000000000000000000000000000000000000000..3630ee039d6976dfe4e27550e4080509797d4410
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/_CHECKPOINT_METADATA
@@ -0,0 +1 @@
+{"init_timestamp_nsecs": 1734887210978521803, "commit_timestamp_nsecs": 1734887226584723944}
\ No newline at end of file
diff --git a/FuSe_PaliVLA_model/15000/action_tokenizer_model_params/_METADATA b/FuSe_PaliVLA_model/15000/action_tokenizer_model_params/_METADATA
new file mode 100644
index 0000000000000000000000000000000000000000..6c9581f1d222b613c95c8fdad52dce38e7d10a7a
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/action_tokenizer_model_params/_METADATA
@@ -0,0 +1 @@
+{"tree_metadata": {"('params',)": {"key_metadata": [{"key": "params", "key_type": 2}], "value_metadata": {"value_type": "Dict", "skip_deserialize": true}}}, "use_zarr3": false}
\ No newline at end of file
diff --git a/FuSe_PaliVLA_model/15000/action_tokenizer_model_params/commit_success.txt b/FuSe_PaliVLA_model/15000/action_tokenizer_model_params/commit_success.txt
new file mode 100644
index 0000000000000000000000000000000000000000..698b1e7aa503945e404ccbb94b162687a48ac4c0
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/action_tokenizer_model_params/commit_success.txt
@@ -0,0 +1 @@
+Checkpoint commit was successful to gs://kyle-checkpoints-eu4/fuse-vla-checkpoints/golden-leaf-2/15000/action_tokenizer_model_params
\ No newline at end of file
diff --git a/FuSe_PaliVLA_model/15000/action_tokenizer_model_spec/commit_success.txt b/FuSe_PaliVLA_model/15000/action_tokenizer_model_spec/commit_success.txt
new file mode 100644
index 0000000000000000000000000000000000000000..749fa81386e2b976ea4b33e52fa26e62b3d9136b
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/action_tokenizer_model_spec/commit_success.txt
@@ -0,0 +1 @@
+Checkpoint commit was successful to gs://kyle-checkpoints-eu4/fuse-vla-checkpoints/golden-leaf-2/15000/action_tokenizer_model_spec
\ No newline at end of file
diff --git a/FuSe_PaliVLA_model/15000/action_tokenizer_model_spec/metadata b/FuSe_PaliVLA_model/15000/action_tokenizer_model_spec/metadata
new file mode 100644
index 0000000000000000000000000000000000000000..8624569e216d039d45d8bc64788c75e632ff0692
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/action_tokenizer_model_spec/metadata
@@ -0,0 +1 @@
+{"__ctor": "palivla.tokenizer.BinActionTokenizer", "config": {"action_dim": 7, "action_horizon": 1, "action_vocab_size": 256, "max_action_value": 2, "min_action_value": -2}}
\ No newline at end of file
diff --git a/FuSe_PaliVLA_model/15000/commit_success.txt b/FuSe_PaliVLA_model/15000/commit_success.txt
new file mode 100644
index 0000000000000000000000000000000000000000..e38f374ae26161d760dac173ee9d850f1fd06841
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/commit_success.txt
@@ -0,0 +1 @@
+Checkpoint commit was successful to gs://kyle-checkpoints-eu4/fuse-vla-checkpoints/golden-leaf-2/15000
\ No newline at end of file
diff --git a/FuSe_PaliVLA_model/15000/config/commit_success.txt b/FuSe_PaliVLA_model/15000/config/commit_success.txt
new file mode 100644
index 0000000000000000000000000000000000000000..ead35b1ac43eb327b14c43179695b99d253c35a5
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/config/commit_success.txt
@@ -0,0 +1 @@
+Checkpoint commit was successful to gs://kyle-checkpoints-eu4/fuse-vla-checkpoints/golden-leaf-2/15000/config
\ No newline at end of file
diff --git a/FuSe_PaliVLA_model/15000/config/metadata b/FuSe_PaliVLA_model/15000/config/metadata
new file mode 100644
index 0000000000000000000000000000000000000000..d64a00142288ff624d0e57ef8ddf7aedceec42ff
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/config/metadata
@@ -0,0 +1 @@
+{"encoder_specs": {"digit_left": {"__ctor": "palivla.tactile_encoder_pooled.tvlViT", "config": {"load_fn": "palivla.utils.load_tvl_weights", "load_kwargs": {"pretrained_path": "gs://oier-europe-bucket/ported_weights/tvl/tvl_vitbgs_params_jax.npz"}, "num_classes": 2048, "pool_type": "cross_attn"}}, "digit_right": {"__ctor": "palivla.tactile_encoder_pooled.tvlViT", "config": {"load_fn": "palivla.utils.load_tvl_weights", "load_kwargs": {"pretrained_path": "gs://oier-europe-bucket/ported_weights/tvl/tvl_vitbgs_params_jax.npz"}, "num_classes": 2048, "pool_type": "cross_attn"}}, "modality_embedder": {"__ctor": "palivla.modality_embedder.ModalityEmbedder", "config": {"dtype_str": "float32", "embedding_dim": 2048, "num_embeddings": 9}}, "spectro": {"__ctor": "octo.model.components.vit_encoders.ResNet26FILM", "config": {"flatten_result": true, "normalize_input": false, "num_classes": 2048, "use_film": false}}}, "img_spec": {"__ctor": "big_vision.models.vit.Model", "config": {"pool_type": "none", "scan": true, "variant": "So400m/14"}}, "llm_spec": {"__ctor": "big_vision.models.proj.paligemma.gemma_bv.Model", "config": {"vocab_size": 257152}}, "modality_mappings": {"image_digit_left": "digit_left", "image_digit_right": "digit_right", "image_primary": "img", "image_wrist": "img", "mel_spectro": "spectro", "modality_idx": "modality_embedder", "text": "llm"}, "prompt_autoregressive": false, "target_key_order": ["image_primary", "image_wrist", "image_digit_left", "image_digit_right", "mel_spectro", "modality_idx"]}
\ No newline at end of file
diff --git a/FuSe_PaliVLA_model/15000/dataset_statistics/commit_success.txt b/FuSe_PaliVLA_model/15000/dataset_statistics/commit_success.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1fd65307f2d49b8a923678c99cb9420dfbeb2abc
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/dataset_statistics/commit_success.txt
@@ -0,0 +1 @@
+Checkpoint commit was successful to gs://kyle-checkpoints-eu4/fuse-vla-checkpoints/golden-leaf-2/15000/dataset_statistics
\ No newline at end of file
diff --git a/FuSe_PaliVLA_model/15000/dataset_statistics/metadata b/FuSe_PaliVLA_model/15000/dataset_statistics/metadata
new file mode 100644
index 0000000000000000000000000000000000000000..fab375607f2dfd88f491277e9d0d09e1fcee05c7
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/dataset_statistics/metadata
@@ -0,0 +1 @@
+{"digit_dataset:54.0.0": {"action": {"max": [0.3012024164199829, 0.08334571868181229, 0.27207261323928833, 2.9595746994018555, 1.7334380149841309, 0.5227872729301453, 1.0], "mean": [0.0036537842825055122, 0.0005640247254632413, -0.0020414511673152447, 0.00011840004299301654, 0.0001269286876777187, 0.002870982512831688, 0.6252029538154602], "min": [-0.30079442262649536, -0.07893040776252747, -0.22207710146903992, -3.0137717723846436, -1.6758967638015747, -0.6511449217796326, 0.0], "p01": [-0.027438574433326722, -0.03466422662138939, -0.04764604061841965, -0.10363286197185516, -0.11471884071826935, -0.2442669266462326, 0.0], "p99": [0.047954644858837124, 0.03571233958005904, 0.05198519051074981, 0.09977680832147597, 0.10635631442070007, 0.25084667444229125, 1.0], "std": [0.013129397295415401, 0.011655371636152267, 0.020389817655086517, 0.03639911115169525, 0.0390063151717186, 0.0826837494969368, 0.4840744137763977]}, "num_trajectories": 26866, "num_transitions": 760689, "proprio": {"max": [0.39125144481658936, 0.21706213057041168, 0.3431672751903534, 0.7583836317062378, 1.570796251296997, 2.261908769607544, 1.1190614700317383], "mean": [0.22091123461723328, 0.0011478941887617111, 0.11123534291982651, -0.018648672848939896, -0.09925685822963715, 0.07714134454727173, 0.837134063243866], "min": [-0.04167502000927925, -0.37576887011528015, -0.13934996724128723, -3.141592502593994, -0.887220561504364, -3.0429744720458984, 0.07624256610870361], "p01": [0.11441598266363144, -0.18062341213226318, 0.010478892847895622, -0.32690849661827087, -0.4588014805316925, -1.3974858283996583, 0.08998419344425201], "p99": [0.3416081404685974, 0.14677867233753203, 0.2438981831073761, 0.2692363321781158, 0.2839945793151855, 1.4195377826690674, 1.003452181816101], "std": [0.06400610506534576, 0.06845647841691971, 0.07493763417005539, 0.11028624325990677, 0.1367144137620926, 0.5591349601745605, 0.26563045382499695]}}}
\ No newline at end of file
diff --git a/FuSe_PaliVLA_model/15000/language_tokenizer/commit_success.txt b/FuSe_PaliVLA_model/15000/language_tokenizer/commit_success.txt
new file mode 100644
index 0000000000000000000000000000000000000000..cc72807eeb5fc6c27608d4813bba47faca155a76
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/language_tokenizer/commit_success.txt
@@ -0,0 +1 @@
+Checkpoint commit was successful to gs://kyle-checkpoints-eu4/fuse-vla-checkpoints/golden-leaf-2/15000/language_tokenizer
\ No newline at end of file
diff --git a/FuSe_PaliVLA_model/15000/language_tokenizer/language_tokenizer.proto b/FuSe_PaliVLA_model/15000/language_tokenizer/language_tokenizer.proto
new file mode 100644
index 0000000000000000000000000000000000000000..a510e3c25e21f5263ac4a60a583c5b42bc454d0f
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/language_tokenizer/language_tokenizer.proto
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d05a0d13faead3332d43844b68b91753eae2e73954cf79fbc3fc0361cb0819d8
+size 15152831
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/_METADATA b/FuSe_PaliVLA_model/15000/model_model_params/_METADATA
new file mode 100644
index 0000000000000000000000000000000000000000..249e50c06ec93d706a75e5188f72ae51ff0324ee
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/_METADATA
@@ -0,0 +1 @@
+{"tree_metadata": {"('params', 'digit_left', 'block0', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block0', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block0', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block0', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block0', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block0', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block0', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block0', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block0', 'norm1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block0', 'norm1', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block0', 'norm2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block0', 'norm2', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block1', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block1', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block1', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block1', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block1', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block1', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block1', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block1', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block1', 'norm1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block1', 'norm1', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block1', 'norm2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block1', 'norm2', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block10', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block10', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block10', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block10', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block10', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block10', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block10', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block10', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block10', 'norm1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block10', 'norm1', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block10', 'norm2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block10', 'norm2', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block11', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block11', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block11', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block11', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block11', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block11', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block11', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block11', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block11', 'norm1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block11', 'norm1', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block11', 'norm2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block11', 'norm2', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block2', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block2', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block2', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block2', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block2', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block2', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block2', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block2', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block2', 'norm1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block2', 'norm1', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block2', 'norm2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block2', 'norm2', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block3', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block3', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block3', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block3', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block3', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block3', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block3', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block3', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block3', 'norm1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block3', 'norm1', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block3', 'norm2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block3', 'norm2', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block4', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block4', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block4', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block4', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block4', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block4', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block4', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block4', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block4', 'norm1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block4', 'norm1', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block4', 'norm2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block4', 'norm2', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block5', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block5', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block5', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block5', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block5', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block5', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block5', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block5', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block5', 'norm1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block5', 'norm1', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block5', 'norm2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block5', 'norm2', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block6', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block6', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block6', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block6', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block6', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block6', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block6', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block6', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block6', 'norm1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block6', 'norm1', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block6', 'norm2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block6', 'norm2', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block7', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block7', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block7', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block7', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block7', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block7', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block7', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block7', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block7', 'norm1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block7', 'norm1', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block7', 'norm2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block7', 'norm2', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block8', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block8', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block8', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block8', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block8', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block8', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block8', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block8', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block8', 'norm1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block8', 'norm1', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block8', 'norm2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block8', 'norm2', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block9', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block9', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block9', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block9', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block9', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block9', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block9', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block9', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block9', 'norm1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block9', 'norm1', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block9', 'norm2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'block9', 'norm2', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'cls_token')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "cls_token", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'fc_norm', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "fc_norm", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'fc_norm', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "fc_norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'head', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "head", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'head', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "head", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'patch_embed', 'proj', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "patch_embed", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'patch_embed', 'proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "patch_embed", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'pool_func', 'attention_head', 'key', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "key", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'pool_func', 'attention_head', 'key', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "key", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'pool_func', 'attention_head', 'out', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "out", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'pool_func', 'attention_head', 'out', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "out", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'pool_func', 'attention_head', 'query', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "query", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'pool_func', 'attention_head', 'query', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "query", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'pool_func', 'attention_head', 'value', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "value", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'pool_func', 'attention_head', 'value', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "value", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'pool_func', 'readouts')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "readouts", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_left', 'pos_embed')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pos_embed", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block0', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block0', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block0', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block0', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block0', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block0', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block0', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block0', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block0', 'norm1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block0', 'norm1', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block0', 'norm2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block0', 'norm2', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block1', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block1', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block1', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block1', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block1', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block1', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block1', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block1', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block1', 'norm1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block1', 'norm1', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block1', 'norm2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block1', 'norm2', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block10', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block10', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block10', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block10', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block10', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block10', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block10', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block10', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block10', 'norm1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block10', 'norm1', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block10', 'norm2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block10', 'norm2', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block11', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block11', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block11', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block11', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block11', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block11', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block11', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block11', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block11', 'norm1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block11', 'norm1', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block11', 'norm2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block11', 'norm2', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block2', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block2', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block2', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block2', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block2', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block2', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block2', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block2', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block2', 'norm1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block2', 'norm1', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block2', 'norm2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block2', 'norm2', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block3', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block3', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block3', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block3', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block3', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block3', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block3', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block3', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block3', 'norm1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block3', 'norm1', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block3', 'norm2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block3', 'norm2', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block4', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block4', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block4', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block4', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block4', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block4', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block4', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block4', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block4', 'norm1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block4', 'norm1', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block4', 'norm2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block4', 'norm2', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block5', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block5', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block5', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block5', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block5', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block5', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block5', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block5', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block5', 'norm1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block5', 'norm1', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block5', 'norm2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block5', 'norm2', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block6', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block6', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block6', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block6', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block6', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block6', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block6', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block6', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block6', 'norm1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block6', 'norm1', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block6', 'norm2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block6', 'norm2', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block7', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block7', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block7', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block7', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block7', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block7', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block7', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block7', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block7', 'norm1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block7', 'norm1', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block7', 'norm2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block7', 'norm2', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block8', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block8', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block8', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block8', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block8', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block8', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block8', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block8', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block8', 'norm1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block8', 'norm1', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block8', 'norm2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block8', 'norm2', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block9', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block9', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block9', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block9', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block9', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block9', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block9', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block9', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block9', 'norm1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block9', 'norm1', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block9', 'norm2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'block9', 'norm2', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'cls_token')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "cls_token", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'fc_norm', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "fc_norm", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'fc_norm', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "fc_norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'head', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "head", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'head', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "head", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'patch_embed', 'proj', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "patch_embed", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'patch_embed', 'proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "patch_embed", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'pool_func', 'attention_head', 'key', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "key", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'pool_func', 'attention_head', 'key', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "key", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'pool_func', 'attention_head', 'out', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "out", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'pool_func', 'attention_head', 'out', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "out", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'pool_func', 'attention_head', 'query', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "query", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'pool_func', 'attention_head', 'query', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "query", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'pool_func', 'attention_head', 'value', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "value", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'pool_func', 'attention_head', 'value', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "value", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'pool_func', 'readouts')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "readouts", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'digit_right', 'pos_embed')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pos_embed", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'img', 'Transformer', 'encoder_norm', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoder_norm", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'img', 'Transformer', 'encoder_norm', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoder_norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'img', 'Transformer', 'encoderblock', 'LayerNorm_0', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "LayerNorm_0", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'img', 'Transformer', 'encoderblock', 'LayerNorm_0', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "LayerNorm_0", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'img', 'Transformer', 'encoderblock', 'LayerNorm_1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "LayerNorm_1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'img', 'Transformer', 'encoderblock', 'LayerNorm_1', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "LayerNorm_1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'img', 'Transformer', 'encoderblock', 'MlpBlock_0', 'Dense_0', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MlpBlock_0", "key_type": 2}, {"key": "Dense_0", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'img', 'Transformer', 'encoderblock', 'MlpBlock_0', 'Dense_0', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MlpBlock_0", "key_type": 2}, {"key": "Dense_0", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'img', 'Transformer', 'encoderblock', 'MlpBlock_0', 'Dense_1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MlpBlock_0", "key_type": 2}, {"key": "Dense_1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'img', 'Transformer', 'encoderblock', 'MlpBlock_0', 'Dense_1', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MlpBlock_0", "key_type": 2}, {"key": "Dense_1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'key', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "key", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'key', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "key", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'out', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "out", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'out', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "out", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'query', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "query", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'query', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "query", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'value', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "value", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'value', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "value", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'img', 'embedding', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "embedding", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'img', 'embedding', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "embedding", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'img', 'head', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "head", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'img', 'head', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "head", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'img', 'pos_embedding')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "pos_embedding", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'llm', 'embedder', 'input_embedding')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "embedder", "key_type": 2}, {"key": "input_embedding", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'llm', 'final_norm', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "final_norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'llm', 'layers', 'attn', 'attn_vec_einsum', 'w')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "attn_vec_einsum", "key_type": 2}, {"key": "w", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'llm', 'layers', 'attn', 'kv_einsum', 'w')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "kv_einsum", "key_type": 2}, {"key": "w", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'llm', 'layers', 'attn', 'q_einsum', 'w')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "q_einsum", "key_type": 2}, {"key": "w", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'llm', 'layers', 'mlp', 'gating_einsum')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "gating_einsum", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'llm', 'layers', 'mlp', 'linear')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "linear", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'llm', 'layers', 'pre_attention_norm', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "pre_attention_norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'llm', 'layers', 'pre_ffw_norm', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "pre_ffw_norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'modality_embedder', 'Embed_0', 'embedding')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "modality_embedder", "key_type": 2}, {"key": "Embed_0", "key_type": 2}, {"key": "embedding", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'Dense_0', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "Dense_0", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'Dense_0', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "Dense_0", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block1', 'unit1', 'conv1', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block1', 'unit1', 'conv2', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block1', 'unit1', 'conv3', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block1', 'unit1', 'conv_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block1', 'unit1', 'gn1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block1', 'unit1', 'gn1', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block1', 'unit1', 'gn2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block1', 'unit1', 'gn2', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block1', 'unit1', 'gn3', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block1', 'unit1', 'gn3', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block1', 'unit1', 'gn_proj', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block1', 'unit1', 'gn_proj', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block1', 'unit2', 'conv1', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block1', 'unit2', 'conv2', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block1', 'unit2', 'conv3', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block1', 'unit2', 'gn1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block1', 'unit2', 'gn1', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block1', 'unit2', 'gn2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block1', 'unit2', 'gn2', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block1', 'unit2', 'gn3', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block1', 'unit2', 'gn3', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block2', 'unit1', 'conv1', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block2', 'unit1', 'conv2', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block2', 'unit1', 'conv3', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block2', 'unit1', 'conv_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block2', 'unit1', 'gn1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block2', 'unit1', 'gn1', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block2', 'unit1', 'gn2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block2', 'unit1', 'gn2', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block2', 'unit1', 'gn3', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block2', 'unit1', 'gn3', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block2', 'unit1', 'gn_proj', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block2', 'unit1', 'gn_proj', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block2', 'unit2', 'conv1', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block2', 'unit2', 'conv2', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block2', 'unit2', 'conv3', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block2', 'unit2', 'gn1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block2', 'unit2', 'gn1', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block2', 'unit2', 'gn2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block2', 'unit2', 'gn2', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block2', 'unit2', 'gn3', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block2', 'unit2', 'gn3', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block3', 'unit1', 'conv1', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block3', 'unit1', 'conv2', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block3', 'unit1', 'conv3', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block3', 'unit1', 'conv_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block3', 'unit1', 'gn1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block3', 'unit1', 'gn1', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block3', 'unit1', 'gn2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block3', 'unit1', 'gn2', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block3', 'unit1', 'gn3', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block3', 'unit1', 'gn3', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block3', 'unit1', 'gn_proj', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block3', 'unit1', 'gn_proj', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block3', 'unit2', 'conv1', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block3', 'unit2', 'conv2', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block3', 'unit2', 'conv3', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block3', 'unit2', 'gn1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block3', 'unit2', 'gn1', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block3', 'unit2', 'gn2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block3', 'unit2', 'gn2', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block3', 'unit2', 'gn3', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block3', 'unit2', 'gn3', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block4', 'unit1', 'conv1', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block4', 'unit1', 'conv2', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block4', 'unit1', 'conv3', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block4', 'unit1', 'conv_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block4', 'unit1', 'gn1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block4', 'unit1', 'gn1', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block4', 'unit1', 'gn2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block4', 'unit1', 'gn2', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block4', 'unit1', 'gn3', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block4', 'unit1', 'gn3', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block4', 'unit1', 'gn_proj', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block4', 'unit1', 'gn_proj', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block4', 'unit2', 'conv1', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block4', 'unit2', 'conv2', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block4', 'unit2', 'conv3', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block4', 'unit2', 'gn1', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block4', 'unit2', 'gn1', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block4', 'unit2', 'gn2', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block4', 'unit2', 'gn2', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block4', 'unit2', 'gn3', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'block4', 'unit2', 'gn3', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'conv_root', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "conv_root", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'gn_root', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "gn_root", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'spectro', 'gn_root', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "gn_root", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'start_image_digit_left')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "start_image_digit_left", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'start_image_digit_right')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "start_image_digit_right", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'start_image_primary')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "start_image_primary", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'start_image_wrist')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "start_image_wrist", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'start_mel_spectro')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "start_mel_spectro", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'start_modality_idx')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "start_modality_idx", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('params', 'start_text')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "start_text", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}}, "use_zarr3": false}
\ No newline at end of file
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/_sharding b/FuSe_PaliVLA_model/15000/model_model_params/_sharding
new file mode 100644
index 0000000000000000000000000000000000000000..7d8702b2be106f61a4bde7008917973485c05eaf
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/_sharding
@@ -0,0 +1 @@
+{"cGFyYW1zLm1vZGFsaXR5X2VtYmVkZGVyLkVtYmVkXzAuZW1iZWRkaW5n":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuY2xzX3Rva2Vu":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s0Lm1scC5mYzEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s0Lm1scC5mYzEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s0Lm1scC5mYzIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s0Lm1scC5mYzIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s0Lm5vcm0xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s0Lm5vcm0xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s0Lm5vcm0yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s0Lm5vcm0yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s0LmF0dG4ucHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s0LmF0dG4ucHJvai5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s0LmF0dG4ucWt2LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s0LmF0dG4ucWt2Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s1Lm1scC5mYzEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s1Lm1scC5mYzEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s1Lm1scC5mYzIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s1Lm1scC5mYzIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s1Lm5vcm0xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s1Lm5vcm0xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s1Lm5vcm0yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s1Lm5vcm0yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s1LmF0dG4ucHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s1LmF0dG4ucHJvai5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s1LmF0dG4ucWt2LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s1LmF0dG4ucWt2Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s2Lm1scC5mYzEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s2Lm1scC5mYzEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s2Lm1scC5mYzIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s2Lm1scC5mYzIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s2Lm5vcm0xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s2Lm5vcm0xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s2Lm5vcm0yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s2Lm5vcm0yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s2LmF0dG4ucHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s2LmF0dG4ucHJvai5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s2LmF0dG4ucWt2LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s2LmF0dG4ucWt2Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s3Lm1scC5mYzEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s3Lm1scC5mYzEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s3Lm1scC5mYzIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s3Lm1scC5mYzIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s3Lm5vcm0xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s3Lm5vcm0xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s3Lm5vcm0yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s3Lm5vcm0yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s3LmF0dG4ucHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s3LmF0dG4ucHJvai5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s3LmF0dG4ucWt2LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s3LmF0dG4ucWt2Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s4Lm1scC5mYzEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s4Lm1scC5mYzEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s4Lm1scC5mYzIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s4Lm1scC5mYzIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s4Lm5vcm0xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s4Lm5vcm0xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s4Lm5vcm0yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s4Lm5vcm0yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s4LmF0dG4ucHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s4LmF0dG4ucHJvai5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s4LmF0dG4ucWt2LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s4LmF0dG4ucWt2Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s5Lm1scC5mYzEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s5Lm1scC5mYzEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s5Lm1scC5mYzIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s5Lm1scC5mYzIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s5Lm5vcm0xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s5Lm5vcm0xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s5Lm5vcm0yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s5Lm5vcm0yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s5LmF0dG4ucHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s5LmF0dG4ucHJvai5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s5LmF0dG4ucWt2LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2s5LmF0dG4ucWt2Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2swLm1scC5mYzEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2swLm1scC5mYzEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2swLm1scC5mYzIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2swLm1scC5mYzIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2swLm5vcm0xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2swLm5vcm0xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2swLm5vcm0yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2swLm5vcm0yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2swLmF0dG4ucHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2swLmF0dG4ucHJvai5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2swLmF0dG4ucWt2LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2swLmF0dG4ucWt2Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2sxLm1scC5mYzEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2sxLm1scC5mYzEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2sxLm1scC5mYzIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2sxLm1scC5mYzIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2sxLm5vcm0xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2sxLm5vcm0xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2sxLm5vcm0yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2sxLm5vcm0yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2sxLmF0dG4ucHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2sxLmF0dG4ucHJvai5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2sxLmF0dG4ucWt2LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2sxLmF0dG4ucWt2Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2sxMC5hdHRuLnByb2ouYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2sxMC5hdHRuLnByb2oua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2sxMC5hdHRuLnFrdi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2sxMC5hdHRuLnFrdi5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2sxMC5tbHAuZmMxLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2sxMC5tbHAuZmMxLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2sxMC5tbHAuZmMyLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2sxMC5tbHAuZmMyLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2sxMC5ub3JtMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2sxMC5ub3JtMS5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2sxMC5ub3JtMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2sxMC5ub3JtMi5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2sxMS5hdHRuLnByb2ouYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2sxMS5hdHRuLnByb2oua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2sxMS5hdHRuLnFrdi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2sxMS5hdHRuLnFrdi5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2sxMS5tbHAuZmMxLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2sxMS5tbHAuZmMxLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2sxMS5tbHAuZmMyLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2sxMS5tbHAuZmMyLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2sxMS5ub3JtMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2sxMS5ub3JtMS5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2sxMS5ub3JtMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2sxMS5ub3JtMi5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2syLm1scC5mYzEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2syLm1scC5mYzEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2syLm1scC5mYzIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2syLm1scC5mYzIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2syLm5vcm0xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2syLm5vcm0xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2syLm5vcm0yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2syLm5vcm0yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2syLmF0dG4ucHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2syLmF0dG4ucHJvai5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2syLmF0dG4ucWt2LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2syLmF0dG4ucWt2Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2szLm1scC5mYzEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2szLm1scC5mYzEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2szLm1scC5mYzIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2szLm1scC5mYzIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2szLm5vcm0xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2szLm5vcm0xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2szLm5vcm0yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2szLm5vcm0yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2szLmF0dG4ucHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2szLmF0dG4ucHJvai5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2szLmF0dG4ucWt2LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuYmxvY2szLmF0dG4ucWt2Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuZmNfbm9ybS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuZmNfbm9ybS5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuaGVhZC5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQuaGVhZC5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQucG9vbF9mdW5jLmF0dGVudGlvbl9oZWFkLm91dC5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQucG9vbF9mdW5jLmF0dGVudGlvbl9oZWFkLm91dC5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQucG9vbF9mdW5jLmF0dGVudGlvbl9oZWFkLmtleS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQucG9vbF9mdW5jLmF0dGVudGlvbl9oZWFkLmtleS5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQucG9vbF9mdW5jLmF0dGVudGlvbl9oZWFkLnF1ZXJ5LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQucG9vbF9mdW5jLmF0dGVudGlvbl9oZWFkLnF1ZXJ5Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQucG9vbF9mdW5jLmF0dGVudGlvbl9oZWFkLnZhbHVlLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQucG9vbF9mdW5jLmF0dGVudGlvbl9oZWFkLnZhbHVlLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQucG9vbF9mdW5jLnJlYWRvdXRz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQucG9zX2VtYmVk":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQucGF0Y2hfZW1iZWQucHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X2xlZnQucGF0Y2hfZW1iZWQucHJvai5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMC5hdHRuLnByb2ouYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMC5hdHRuLnByb2oua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMC5hdHRuLnFrdi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMC5hdHRuLnFrdi5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMC5tbHAuZmMxLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMC5tbHAuZmMxLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMC5tbHAuZmMyLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMC5tbHAuZmMyLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMC5ub3JtMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMC5ub3JtMS5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMC5ub3JtMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMC5ub3JtMi5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMS5hdHRuLnByb2ouYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMS5hdHRuLnByb2oua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMS5hdHRuLnFrdi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMS5hdHRuLnFrdi5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMS5tbHAuZmMxLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMS5tbHAuZmMxLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMS5tbHAuZmMyLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMS5tbHAuZmMyLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMS5ub3JtMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMS5ub3JtMS5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMS5ub3JtMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMS5ub3JtMi5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMTAuYXR0bi5wcm9qLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMTAuYXR0bi5wcm9qLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMTAuYXR0bi5xa3YuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMTAuYXR0bi5xa3Yua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMTAubWxwLmZjMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMTAubWxwLmZjMS5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMTAubWxwLmZjMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMTAubWxwLmZjMi5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMTAubm9ybTEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMTAubm9ybTEuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMTAubm9ybTIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMTAubm9ybTIuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMTEuYXR0bi5wcm9qLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMTEuYXR0bi5wcm9qLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMTEuYXR0bi5xa3YuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMTEuYXR0bi5xa3Yua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMTEubWxwLmZjMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMTEubWxwLmZjMS5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMTEubWxwLmZjMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMTEubWxwLmZjMi5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMTEubm9ybTEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMTEubm9ybTEuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMTEubm9ybTIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMTEubm9ybTIuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMi5hdHRuLnByb2ouYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMi5hdHRuLnByb2oua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMi5hdHRuLnFrdi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMi5hdHRuLnFrdi5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMi5tbHAuZmMxLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMi5tbHAuZmMxLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMi5tbHAuZmMyLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMi5tbHAuZmMyLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMi5ub3JtMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMi5ub3JtMS5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMi5ub3JtMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMi5ub3JtMi5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMy5hdHRuLnByb2ouYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMy5hdHRuLnByb2oua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMy5hdHRuLnFrdi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMy5hdHRuLnFrdi5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMy5tbHAuZmMxLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMy5tbHAuZmMxLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMy5tbHAuZmMyLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMy5tbHAuZmMyLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMy5ub3JtMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMy5ub3JtMS5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMy5ub3JtMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrMy5ub3JtMi5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNC5hdHRuLnByb2ouYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNC5hdHRuLnByb2oua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNC5hdHRuLnFrdi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNC5hdHRuLnFrdi5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNC5tbHAuZmMxLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNC5tbHAuZmMxLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNC5tbHAuZmMyLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNC5tbHAuZmMyLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNC5ub3JtMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNC5ub3JtMS5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNC5ub3JtMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNC5ub3JtMi5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNS5hdHRuLnByb2ouYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNS5hdHRuLnByb2oua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNS5hdHRuLnFrdi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNS5hdHRuLnFrdi5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNS5tbHAuZmMxLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNS5tbHAuZmMxLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNS5tbHAuZmMyLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNS5tbHAuZmMyLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNS5ub3JtMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNS5ub3JtMS5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNS5ub3JtMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNS5ub3JtMi5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNi5hdHRuLnByb2ouYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNi5hdHRuLnByb2oua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNi5hdHRuLnFrdi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNi5hdHRuLnFrdi5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNi5tbHAuZmMxLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNi5tbHAuZmMxLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNi5tbHAuZmMyLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNi5tbHAuZmMyLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNi5ub3JtMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNi5ub3JtMS5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNi5ub3JtMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNi5ub3JtMi5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNy5hdHRuLnByb2ouYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNy5hdHRuLnByb2oua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNy5hdHRuLnFrdi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNy5hdHRuLnFrdi5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNy5tbHAuZmMxLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNy5tbHAuZmMxLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNy5tbHAuZmMyLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNy5tbHAuZmMyLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNy5ub3JtMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNy5ub3JtMS5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNy5ub3JtMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrNy5ub3JtMi5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrOC5hdHRuLnByb2ouYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrOC5hdHRuLnByb2oua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrOC5hdHRuLnFrdi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrOC5hdHRuLnFrdi5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrOC5tbHAuZmMxLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrOC5tbHAuZmMxLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrOC5tbHAuZmMyLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrOC5tbHAuZmMyLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrOC5ub3JtMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrOC5ub3JtMS5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrOC5ub3JtMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrOC5ub3JtMi5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrOS5hdHRuLnByb2ouYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrOS5hdHRuLnByb2oua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrOS5hdHRuLnFrdi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrOS5hdHRuLnFrdi5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrOS5tbHAuZmMxLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrOS5tbHAuZmMxLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrOS5tbHAuZmMyLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrOS5tbHAuZmMyLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrOS5ub3JtMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrOS5ub3JtMS5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrOS5ub3JtMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmJsb2NrOS5ub3JtMi5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmNsc190b2tlbg==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmZjX25vcm0uYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmZjX25vcm0uc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmhlYWQuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LmhlYWQua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LnBhdGNoX2VtYmVkLnByb2ouYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LnBhdGNoX2VtYmVkLnByb2oua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LnBvb2xfZnVuYy5hdHRlbnRpb25faGVhZC52YWx1ZS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LnBvb2xfZnVuYy5hdHRlbnRpb25faGVhZC52YWx1ZS5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LnBvb2xfZnVuYy5hdHRlbnRpb25faGVhZC5rZXkuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LnBvb2xfZnVuYy5hdHRlbnRpb25faGVhZC5rZXkua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LnBvb2xfZnVuYy5hdHRlbnRpb25faGVhZC5vdXQuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LnBvb2xfZnVuYy5hdHRlbnRpb25faGVhZC5vdXQua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LnBvb2xfZnVuYy5hdHRlbnRpb25faGVhZC5xdWVyeS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LnBvb2xfZnVuYy5hdHRlbnRpb25faGVhZC5xdWVyeS5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LnBvb2xfZnVuYy5yZWFkb3V0cw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmRpZ2l0X3JpZ2h0LnBvc19lbWJlZA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyX25vcm0uYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyX25vcm0uc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTGF5ZXJOb3JtXzAuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTGF5ZXJOb3JtXzAuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTGF5ZXJOb3JtXzEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTGF5ZXJOb3JtXzEuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTWxwQmxvY2tfMC5EZW5zZV8wLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTWxwQmxvY2tfMC5EZW5zZV8wLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, \"fsdp\", null]}","cGFyYW1zLmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTWxwQmxvY2tfMC5EZW5zZV8xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTWxwQmxvY2tfMC5EZW5zZV8xLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, null, \"fsdp\"]}","cGFyYW1zLmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTXVsdGlIZWFkRG90UHJvZHVjdEF0dGVudGlvbl8wLm91dC5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTXVsdGlIZWFkRG90UHJvZHVjdEF0dGVudGlvbl8wLm91dC5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, null, null, \"fsdp\"]}","cGFyYW1zLmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTXVsdGlIZWFkRG90UHJvZHVjdEF0dGVudGlvbl8wLmtleS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTXVsdGlIZWFkRG90UHJvZHVjdEF0dGVudGlvbl8wLmtleS5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, \"fsdp\", null, null]}","cGFyYW1zLmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTXVsdGlIZWFkRG90UHJvZHVjdEF0dGVudGlvbl8wLnF1ZXJ5LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTXVsdGlIZWFkRG90UHJvZHVjdEF0dGVudGlvbl8wLnF1ZXJ5Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, \"fsdp\", null, null]}","cGFyYW1zLmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTXVsdGlIZWFkRG90UHJvZHVjdEF0dGVudGlvbl8wLnZhbHVlLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTXVsdGlIZWFkRG90UHJvZHVjdEF0dGVudGlvbl8wLnZhbHVlLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, \"fsdp\", null, null]}","cGFyYW1zLmltZy5lbWJlZGRpbmcuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmltZy5lbWJlZGRpbmcua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmltZy5oZWFkLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmltZy5oZWFkLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, \"fsdp\"]}","cGFyYW1zLmltZy5wb3NfZW1iZWRkaW5n":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmxsbS5lbWJlZGRlci5pbnB1dF9lbWJlZGRpbmc=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [\"fsdp\", null]}","cGFyYW1zLmxsbS5maW5hbF9ub3JtLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmxsbS5sYXllcnMuYXR0bi5hdHRuX3ZlY19laW5zdW0udw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, null, null, \"fsdp\"]}","cGFyYW1zLmxsbS5sYXllcnMuYXR0bi5rdl9laW5zdW0udw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, null, null, \"fsdp\", null]}","cGFyYW1zLmxsbS5sYXllcnMuYXR0bi5xX2VpbnN1bS53":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, null, \"fsdp\", null]}","cGFyYW1zLmxsbS5sYXllcnMubWxwLmdhdGluZ19laW5zdW0=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, null, null, \"fsdp\"]}","cGFyYW1zLmxsbS5sYXllcnMubWxwLmxpbmVhcg==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, \"fsdp\", null]}","cGFyYW1zLmxsbS5sYXllcnMucHJlX2F0dGVudGlvbl9ub3JtLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLmxsbS5sYXllcnMucHJlX2Zmd19ub3JtLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnN0YXJ0X21lbF9zcGVjdHJv":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnN0YXJ0X21vZGFsaXR5X2lkeA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnN0YXJ0X2ltYWdlX2RpZ2l0X2xlZnQ=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnN0YXJ0X2ltYWdlX2RpZ2l0X3JpZ2h0":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnN0YXJ0X2ltYWdlX3ByaW1hcnk=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnN0YXJ0X2ltYWdlX3dyaXN0":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnN0YXJ0X3RleHQ=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uRGVuc2VfMC5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uRGVuc2VfMC5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [\"fsdp\", null]}","cGFyYW1zLnNwZWN0cm8uY29udl9yb290Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2s0LnVuaXQxLmNvbnYxLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2s0LnVuaXQxLmNvbnYyLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, null, \"fsdp\", null]}","cGFyYW1zLnNwZWN0cm8uYmxvY2s0LnVuaXQxLmNvbnYzLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, null, null, \"fsdp\"]}","cGFyYW1zLnNwZWN0cm8uYmxvY2s0LnVuaXQxLmNvbnZfcHJvai5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, null, null, \"fsdp\"]}","cGFyYW1zLnNwZWN0cm8uYmxvY2s0LnVuaXQxLmduMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2s0LnVuaXQxLmduMS5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2s0LnVuaXQxLmduMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2s0LnVuaXQxLmduMi5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2s0LnVuaXQxLmduMy5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2s0LnVuaXQxLmduMy5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2s0LnVuaXQxLmduX3Byb2ouYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2s0LnVuaXQxLmduX3Byb2ouc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2s0LnVuaXQyLmNvbnYxLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, null, \"fsdp\", null]}","cGFyYW1zLnNwZWN0cm8uYmxvY2s0LnVuaXQyLmNvbnYyLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, null, \"fsdp\", null]}","cGFyYW1zLnNwZWN0cm8uYmxvY2s0LnVuaXQyLmNvbnYzLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, null, null, \"fsdp\"]}","cGFyYW1zLnNwZWN0cm8uYmxvY2s0LnVuaXQyLmduMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2s0LnVuaXQyLmduMS5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2s0LnVuaXQyLmduMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2s0LnVuaXQyLmduMi5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2s0LnVuaXQyLmduMy5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2s0LnVuaXQyLmduMy5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2sxLnVuaXQxLmNvbnYxLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2sxLnVuaXQxLmNvbnYyLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2sxLnVuaXQxLmNvbnYzLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2sxLnVuaXQxLmNvbnZfcHJvai5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2sxLnVuaXQxLmduMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2sxLnVuaXQxLmduMS5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2sxLnVuaXQxLmduMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2sxLnVuaXQxLmduMi5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2sxLnVuaXQxLmduMy5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2sxLnVuaXQxLmduMy5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2sxLnVuaXQxLmduX3Byb2ouYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2sxLnVuaXQxLmduX3Byb2ouc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2sxLnVuaXQyLmNvbnYxLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2sxLnVuaXQyLmNvbnYyLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2sxLnVuaXQyLmNvbnYzLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2sxLnVuaXQyLmduMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2sxLnVuaXQyLmduMS5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2sxLnVuaXQyLmduMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2sxLnVuaXQyLmduMi5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2sxLnVuaXQyLmduMy5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2sxLnVuaXQyLmduMy5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2syLnVuaXQxLmNvbnYxLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2syLnVuaXQxLmNvbnYyLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2syLnVuaXQxLmNvbnYzLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2syLnVuaXQxLmNvbnZfcHJvai5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2syLnVuaXQxLmduMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2syLnVuaXQxLmduMS5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2syLnVuaXQxLmduMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2syLnVuaXQxLmduMi5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2syLnVuaXQxLmduMy5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2syLnVuaXQxLmduMy5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2syLnVuaXQxLmduX3Byb2ouYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2syLnVuaXQxLmduX3Byb2ouc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2syLnVuaXQyLmNvbnYxLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2syLnVuaXQyLmNvbnYyLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2syLnVuaXQyLmNvbnYzLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2syLnVuaXQyLmduMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2syLnVuaXQyLmduMS5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2syLnVuaXQyLmduMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2syLnVuaXQyLmduMi5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2syLnVuaXQyLmduMy5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2syLnVuaXQyLmduMy5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2szLnVuaXQxLmNvbnYxLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2szLnVuaXQxLmNvbnYyLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2szLnVuaXQxLmNvbnYzLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2szLnVuaXQxLmNvbnZfcHJvai5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2szLnVuaXQxLmduMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2szLnVuaXQxLmduMS5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2szLnVuaXQxLmduMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2szLnVuaXQxLmduMi5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2szLnVuaXQxLmduMy5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2szLnVuaXQxLmduMy5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2szLnVuaXQxLmduX3Byb2ouYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2szLnVuaXQxLmduX3Byb2ouc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2szLnVuaXQyLmNvbnYxLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2szLnVuaXQyLmNvbnYyLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2szLnVuaXQyLmNvbnYzLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2szLnVuaXQyLmduMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2szLnVuaXQyLmduMS5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2szLnVuaXQyLmduMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2szLnVuaXQyLmduMi5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2szLnVuaXQyLmduMy5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uYmxvY2szLnVuaXQyLmduMy5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uZ25fcm9vdC5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","cGFyYW1zLnNwZWN0cm8uZ25fcm9vdC5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}"}
\ No newline at end of file
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/commit_success.txt b/FuSe_PaliVLA_model/15000/model_model_params/commit_success.txt
new file mode 100644
index 0000000000000000000000000000000000000000..9a385e9ffca027e18a070cbded9e8750939b0240
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/commit_success.txt
@@ -0,0 +1 @@
+Checkpoint commit was successful to gs://kyle-checkpoints-eu4/fuse-vla-checkpoints/golden-leaf-2/15000/model_model_params
\ No newline at end of file
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/d/7fa51357ef122322d701ff54656b4044 b/FuSe_PaliVLA_model/15000/model_model_params/d/7fa51357ef122322d701ff54656b4044
new file mode 100644
index 0000000000000000000000000000000000000000..77f940ddbb30026bee4377d88e0c15782d396ffb
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_model_params/d/7fa51357ef122322d701ff54656b4044 differ
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_model_params/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..b8f3ffce9203ad61a3963578e46acff017747044
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_model_params/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_0/d/172a3a8da0873d619cea31d574d4b8ee b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_0/d/172a3a8da0873d619cea31d574d4b8ee
new file mode 100644
index 0000000000000000000000000000000000000000..fe203e9f16715e801c14f7183dbc447a7ae4135c
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_0/d/172a3a8da0873d619cea31d574d4b8ee differ
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_0/d/5c587fdf05b2d8c8ff1dc39cf156148e b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_0/d/5c587fdf05b2d8c8ff1dc39cf156148e
new file mode 100644
index 0000000000000000000000000000000000000000..7cdbe17170b496d0fbba24b4e3974c44d95968d5
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_0/d/5c587fdf05b2d8c8ff1dc39cf156148e differ
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_0/d/5c936ff6c74dc14c3cc9c591c2ce8f20 b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_0/d/5c936ff6c74dc14c3cc9c591c2ce8f20
new file mode 100644
index 0000000000000000000000000000000000000000..e849a531d833b07e4081fae8a6f34a06b5d4457d
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_0/d/5c936ff6c74dc14c3cc9c591c2ce8f20 differ
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_0/d/7b5c971a7e7e6fe913bcc17b374e0f55 b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_0/d/7b5c971a7e7e6fe913bcc17b374e0f55
new file mode 100644
index 0000000000000000000000000000000000000000..3d62159f56ad6d10677e0c42a0157c0587f0c84d
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_0/d/7b5c971a7e7e6fe913bcc17b374e0f55 differ
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_0/d/85bdac45edbd6c9198be15243b619a78 b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_0/d/85bdac45edbd6c9198be15243b619a78
new file mode 100644
index 0000000000000000000000000000000000000000..ca2853c7aa55a93d001bdadf91afb794d965d353
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_0/d/85bdac45edbd6c9198be15243b619a78
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb23cc3bb0d328c0a3c427621c21e8e789c4ca7717e177f796416a88a5f27c7e
+size 522006874
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_0/d/976003b263b86a7e904f54c7dcba50f4 b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_0/d/976003b263b86a7e904f54c7dcba50f4
new file mode 100644
index 0000000000000000000000000000000000000000..389b060554c8dcc6f412233807ce2394eb7cfefd
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_0/d/976003b263b86a7e904f54c7dcba50f4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cffb24caef5335fa15332df688563b5b24a7bc0f19270d99b396ce7a868037e8
+size 8768604
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_0/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_0/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..68a9e58ab5397b26fd690b59fff4a71d22b6f3dd
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_0/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_1/d/26d761b04111868f51482bad86328227 b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_1/d/26d761b04111868f51482bad86328227
new file mode 100644
index 0000000000000000000000000000000000000000..8beefa4c4eb13e7d710f53e37f4e5e728197d00c
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_1/d/26d761b04111868f51482bad86328227
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d141c8f0ea68ffd59416f584c3032889b6e4a90cad0be6dc7f0584add67acc59
+size 286650197
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_1/d/9596eae70329fba3ab7a8617ee74d986 b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_1/d/9596eae70329fba3ab7a8617ee74d986
new file mode 100644
index 0000000000000000000000000000000000000000..77af98f7eedca2f07730e625feac782be1fb5e13
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_1/d/9596eae70329fba3ab7a8617ee74d986
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:54189e5d316aaaf989ab30c6722f8d20aabb5fd51df91206b699a50fae566941
+size 53538355
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_1/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_1/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..987034d059eed247f24e55b08138827e257a996a
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_1/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_10/d/16b31b4c6b474856951719672e63b778 b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_10/d/16b31b4c6b474856951719672e63b778
new file mode 100644
index 0000000000000000000000000000000000000000..048aefbc2d859c6abbeb76695add456c418cb7f1
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_10/d/16b31b4c6b474856951719672e63b778
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d73f65277cfbb37065c57058ef1741ad8a6074e6e913db25bfac0349753c3b59
+size 62661492
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_10/d/1733100935709df605fcf9878094b9c3 b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_10/d/1733100935709df605fcf9878094b9c3
new file mode 100644
index 0000000000000000000000000000000000000000..4cd295eeaa0d8275ddb63cf687cbb67acdbf4a9f
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_10/d/1733100935709df605fcf9878094b9c3
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4a27acabb07c92bff438e2dcb82600174915407a7d6acadf4a6fb722670a7c4
+size 277269387
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_10/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_10/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..de5b014888d99ee101d3abeee466080bad6a3c56
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_10/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_11/d/b2b557800d0ee055927d1a44b9af3eda b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_11/d/b2b557800d0ee055927d1a44b9af3eda
new file mode 100644
index 0000000000000000000000000000000000000000..c2d6402ddadc50f999b7638ddc56ce4657841613
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_11/d/b2b557800d0ee055927d1a44b9af3eda
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e66091cfdae5788f5eb22c4a482b2eb21d4db83bf2673d89841325cd5250b24d
+size 53539310
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_11/d/f54bc57c4228488cfdb02842f3b418b7 b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_11/d/f54bc57c4228488cfdb02842f3b418b7
new file mode 100644
index 0000000000000000000000000000000000000000..5e7d1776a2f677f9eeb12f464e970a795a04a503
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_11/d/f54bc57c4228488cfdb02842f3b418b7
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97ccf978c1bff60c5c2c6dd1309b0c382f634a91728ca78cab4f011ed8bd2270
+size 286356008
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_11/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_11/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..b4ef68e901d33ceabd4a230c21c68ee2a1dfd4e2
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_11/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_12/d/6cdcc9ab301a7f48e694d1aad2c111af b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_12/d/6cdcc9ab301a7f48e694d1aad2c111af
new file mode 100644
index 0000000000000000000000000000000000000000..53c7c9ac6c9f6669baf25be54ce6604d0f70feb5
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_12/d/6cdcc9ab301a7f48e694d1aad2c111af
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8704ac2ed7e392651192b47407ae208fc5ec83dd8c43cd53b9655d61165c686
+size 64487716
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_12/d/8291fad89e6470abe05f5d7d293a35a3 b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_12/d/8291fad89e6470abe05f5d7d293a35a3
new file mode 100644
index 0000000000000000000000000000000000000000..90a61d24024a8bc75ffa1ab71a603b00352ea74f
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_12/d/8291fad89e6470abe05f5d7d293a35a3
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:96b4554d0b9995c4e40b3ac0cebef994e8cf990b7637f92cf684b1e85db60e38
+size 275828292
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_12/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_12/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..cd70d7eaea17a3396ebffc4e32ebcfdd184550dc
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_12/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_13/d/10c08f2bc35f069aa05ca0047138da8e b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_13/d/10c08f2bc35f069aa05ca0047138da8e
new file mode 100644
index 0000000000000000000000000000000000000000..3686fa8509d9f2452d993284fd4efc581ea5145b
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_13/d/10c08f2bc35f069aa05ca0047138da8e
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:781ae13801ec8ab99caa61031de72aff23c8d3c7effffa4963ec8bd06e5434ee
+size 58822552
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_13/d/1d1e71a9ffa1eb758725987c74a464e3 b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_13/d/1d1e71a9ffa1eb758725987c74a464e3
new file mode 100644
index 0000000000000000000000000000000000000000..bff6b14283a3f87890e49e6ecef53cec820e5115
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_13/d/1d1e71a9ffa1eb758725987c74a464e3
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c322449d3b7e6dafae3efcd0bcf1733115e9753467f57b9934aa0dabaa955164
+size 281317518
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_13/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_13/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..5ddf0d809efa0df9b13d82d650d8c4534cb1967a
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_13/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_14/d/a87993cacbecc6bb164093187da985c9 b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_14/d/a87993cacbecc6bb164093187da985c9
new file mode 100644
index 0000000000000000000000000000000000000000..030472ab147968f3de8f7e189103f05d8fc4c106
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_14/d/a87993cacbecc6bb164093187da985c9
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d1cdbeaeee30f1981b08790621554b07fb717fcd117eb488dfbe55fe7e82881
+size 277630487
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_14/d/b969d167d8760c9e5a0e1d30a2cb09fd b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_14/d/b969d167d8760c9e5a0e1d30a2cb09fd
new file mode 100644
index 0000000000000000000000000000000000000000..e9d1a6a26793ae55ebe073f70c4e5fa9275fa6dc
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_14/d/b969d167d8760c9e5a0e1d30a2cb09fd
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44dc516b439d73938c6b82be5ebf1690acaa61a2d3afff772527bf063789b4d4
+size 62305765
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_14/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_14/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..5eb60ffb1537638c7a2ecb1b27ee5377335d62eb
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_14/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_15/d/0bc1e2d4fa3e10b8a5c60b904a3759f9 b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_15/d/0bc1e2d4fa3e10b8a5c60b904a3759f9
new file mode 100644
index 0000000000000000000000000000000000000000..92e4b552075115e2ed80bdb63f02929e267ffa1a
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_15/d/0bc1e2d4fa3e10b8a5c60b904a3759f9
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67789ac412c87cac48048b304646e21368d340e068f0253837215615586b3fb2
+size 58841844
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_15/d/d34decd5ceac0c14e70ec8d4cbc86778 b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_15/d/d34decd5ceac0c14e70ec8d4cbc86778
new file mode 100644
index 0000000000000000000000000000000000000000..238dfd9352e0acda92d4e9708e0c3f3e92976593
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_15/d/d34decd5ceac0c14e70ec8d4cbc86778
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5419cf663baaa96e125fdc86aff79e5eba38145bebdda972d279c715aa4e82ef
+size 281104767
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_15/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_15/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..3e26ad56158ffddef696ea6b11c064ecd8a8ee84
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_15/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_16/d/3677e1699d7ae709527dd2e1477141e7 b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_16/d/3677e1699d7ae709527dd2e1477141e7
new file mode 100644
index 0000000000000000000000000000000000000000..69694e44b5d2bef93fe6047a2930a4bc0b76554c
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_16/d/3677e1699d7ae709527dd2e1477141e7
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:447633b51b374ab10d42a89c72ea0a69894a5c45f056fa1b5ad848991c1caf5d
+size 277103609
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_16/d/d0371996f38251808fe6415d56841d7d b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_16/d/d0371996f38251808fe6415d56841d7d
new file mode 100644
index 0000000000000000000000000000000000000000..73173412d2a47cdc287232f2e5ee9e7e023ffe7d
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_16/d/d0371996f38251808fe6415d56841d7d
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28695d69066f4a41e52a894d8834df5120265086e65799aeb1e403d89e658a5a
+size 63224653
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_16/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_16/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..e82aaf8b354ed9119b286ad1a43b9c8b4fb96231
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_16/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_17/d/74374fad4fa3c3e38ebd5ef167ae059b b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_17/d/74374fad4fa3c3e38ebd5ef167ae059b
new file mode 100644
index 0000000000000000000000000000000000000000..9ea7d5c79e0af6bb83e0eccc7834da36fd3e665c
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_17/d/74374fad4fa3c3e38ebd5ef167ae059b
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d808b1d97034da8d246a9882cd980d098f2f3975707f431e764a636d10279cfe
+size 284403889
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_17/d/bdfd2ec86e0e010f6e71d523352a943f b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_17/d/bdfd2ec86e0e010f6e71d523352a943f
new file mode 100644
index 0000000000000000000000000000000000000000..fea4b5e791b1ecd8f47d26ac0601d8dc10b3223b
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_17/d/bdfd2ec86e0e010f6e71d523352a943f
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f2fd8eeacb5e3acadfdcf2966d5a11eea823143de7387bc4daef2feb6a6d77d
+size 55730401
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_17/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_17/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..d1e4413eac720a5757eb584ec3fc9b604dcbfbfa
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_17/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_18/d/64e689563147cf277b989a13a59c7bfc b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_18/d/64e689563147cf277b989a13a59c7bfc
new file mode 100644
index 0000000000000000000000000000000000000000..f157f7094327a3a855fd706678b639ea956c4055
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_18/d/64e689563147cf277b989a13a59c7bfc
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32d783e63423d4d64678c4d84ebb09d54adc9b617122360e69c7ea43a6f84830
+size 53534494
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_18/d/a1ebcd8a00e0f80c2fbc262ab6ad00bd b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_18/d/a1ebcd8a00e0f80c2fbc262ab6ad00bd
new file mode 100644
index 0000000000000000000000000000000000000000..0b4b6e3c45e80161d46ba1145858569ae736fd71
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_18/d/a1ebcd8a00e0f80c2fbc262ab6ad00bd
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f90107a18a8badc350f1001147dcdd34f7c30e4f29ef666d0f7319b36f1ce3b
+size 286380158
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_18/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_18/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..0104a65699fbf13bf214483499f56a4f69af70a1
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_18/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_19/d/ad4cf83a12cd3b56fac3522c2f60ad10 b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_19/d/ad4cf83a12cd3b56fac3522c2f60ad10
new file mode 100644
index 0000000000000000000000000000000000000000..0b762c238dc71f3d7b982acdc08feeab8cfc4397
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_19/d/ad4cf83a12cd3b56fac3522c2f60ad10
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e252d6f64ce1dc3e086d13a4a3f0546115b6351759c110602f073bce02545dea
+size 283715378
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_19/d/aeb56057d360535456c11e307a73d49e b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_19/d/aeb56057d360535456c11e307a73d49e
new file mode 100644
index 0000000000000000000000000000000000000000..6127b71b640815beab68664051020cf03a8c6f0a
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_19/d/aeb56057d360535456c11e307a73d49e
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aae015b1582eda7df7fea9990db7254c9225005e074ce9ee4d546ae372ec6306
+size 55731751
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_19/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_19/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..f588e993b2812d003c5b2c0d35124b1c57281179
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_19/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_2/d/1d38672b8446b298b569bc9ad19e2a9a b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_2/d/1d38672b8446b298b569bc9ad19e2a9a
new file mode 100644
index 0000000000000000000000000000000000000000..ab832b19954fc8081e564b132ac2e498fce473f7
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_2/d/1d38672b8446b298b569bc9ad19e2a9a
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b17aec9b90dec3cc9a4ada55c3d9135b4dda1a353421b4e6090b0738ee5b2b3f
+size 286440121
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_2/d/5fcdfb031d23ea89302996f02317f27c b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_2/d/5fcdfb031d23ea89302996f02317f27c
new file mode 100644
index 0000000000000000000000000000000000000000..0d626114817f2224994dec7e2e43cf51f608a990
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_2/d/5fcdfb031d23ea89302996f02317f27c
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8a4a298e490ada99e79055317961a4db6b29c8f4dc4627e456cfdc6492219ad
+size 53556294
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_2/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_2/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..2e5c40048099249047b8ed90c6651ddbb542fc51
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_2/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_20/d/2b6cf96aa2e47c9c97fe2ae66e007cb2 b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_20/d/2b6cf96aa2e47c9c97fe2ae66e007cb2
new file mode 100644
index 0000000000000000000000000000000000000000..976e96c1a6f793995f0a2a35aa94d8dd841890ed
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_20/d/2b6cf96aa2e47c9c97fe2ae66e007cb2
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4418bf4410a3319c76afa0c47c452b33b6c87a79bd437d1dedfe90b393723776
+size 57927609
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_20/d/8d4498d6909c0b528fd53a0feaf0b3fc b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_20/d/8d4498d6909c0b528fd53a0feaf0b3fc
new file mode 100644
index 0000000000000000000000000000000000000000..cdcba9ea1ccf6efc889964568caad27e1d662a42
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_20/d/8d4498d6909c0b528fd53a0feaf0b3fc
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2132286c9307cd222a387b18a7a24430abbdbf6ddf029d77fc677506f8f0a47
+size 282392588
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_20/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_20/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..11ffb044a750ed6e6f1b0e172ccfc91754f6a41f
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_20/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_21/d/1a7a94049854b82ccf2b4d7b58bf04bd b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_21/d/1a7a94049854b82ccf2b4d7b58bf04bd
new file mode 100644
index 0000000000000000000000000000000000000000..10e6d9cde6795cdce4ee46d0378e231c376244b9
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_21/d/1a7a94049854b82ccf2b4d7b58bf04bd
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95cde1adc6e69f477a2f6550b91b1cfb11978fa5050519a5fe78e6e2b7c379d6
+size 53545247
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_21/d/8a7f120b1cf3cbfcfba433bad6f6dc77 b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_21/d/8a7f120b1cf3cbfcfba433bad6f6dc77
new file mode 100644
index 0000000000000000000000000000000000000000..e5c4ad8d0587f856a97d446adf521309138637bc
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_21/d/8a7f120b1cf3cbfcfba433bad6f6dc77
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3aaca1cf70d13eea33c2dd9e594ed9eaf26530f05db1fe67014bd699e02581d0
+size 286610792
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_21/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_21/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..08bc3317d86cd9aa13ee1e426fe0b0cfe6e09b22
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_21/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_22/d/104320b48da2ee12a396f83c59100e9e b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_22/d/104320b48da2ee12a396f83c59100e9e
new file mode 100644
index 0000000000000000000000000000000000000000..e916be19234dce374d93e9bdab9d384d788c7ecb
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_22/d/104320b48da2ee12a396f83c59100e9e
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd4c8f4b02d80e9fd7bd39210bb1586b7b5b3bbc9864cc6a7f46de2f5fc5e9cf
+size 222808431
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_22/d/37afc7a232db2d2bec5277b20748e650 b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_22/d/37afc7a232db2d2bec5277b20748e650
new file mode 100644
index 0000000000000000000000000000000000000000..7e740b0e404fdf21a37aec2115e4b28935d81190
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_22/d/37afc7a232db2d2bec5277b20748e650
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:debf4ae9ebf2a47336798229c8f53d6420d8bb126b6bd841b7f5c39fdb53dd36
+size 117084822
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_22/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_22/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..25c1425d2f87ad22d555b9bf38f6fe0f08b137d2
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_22/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_23/d/91a858e355267f36d2e4da81458a20a6 b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_23/d/91a858e355267f36d2e4da81458a20a6
new file mode 100644
index 0000000000000000000000000000000000000000..39c34f8d4e44ad93c6c20585bcbbb6a57e6be6e7
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_23/d/91a858e355267f36d2e4da81458a20a6
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca491a1c985890bfe9ad5fbc1e54e6bc6413247a5b1c2a1b3630241e9e75f4aa
+size 57924990
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_23/d/d528bf5395ffe5094194b1d84dab8b4e b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_23/d/d528bf5395ffe5094194b1d84dab8b4e
new file mode 100644
index 0000000000000000000000000000000000000000..2c0d48389e3da6dde160fa7263b636c5891166fd
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_23/d/d528bf5395ffe5094194b1d84dab8b4e
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e80913c249ee8fdc2fb1c6cf5957804f3c1ef094657da5cae8878822961d5dc8
+size 280234298
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_23/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_23/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..f85af3f8bc8cb03f406344455543e25ecf47ff61
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_23/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_24/d/17060105b41c035fcdb3de23ec5fe55b b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_24/d/17060105b41c035fcdb3de23ec5fe55b
new file mode 100644
index 0000000000000000000000000000000000000000..c9b215a2da4ea8a188b333e4a3173d3c17a18f54
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_24/d/17060105b41c035fcdb3de23ec5fe55b
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2dc7e9b47df3107d0a84f79d20878a2211b2e6ce24919f97b4b4961900ec035
+size 273260543
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_24/d/eea47ac04671d41dffd1a53e4d91aafe b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_24/d/eea47ac04671d41dffd1a53e4d91aafe
new file mode 100644
index 0000000000000000000000000000000000000000..8ff7dfdd819f3beafb9e835f2f9a5d63bf63a07e
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_24/d/eea47ac04671d41dffd1a53e4d91aafe
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b22a226cb0c2ba0c5fa326f98a4ec698e06f068d16d02658e8c993ee5ba0784b
+size 67039994
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_24/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_24/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..54bf1ae7448e61a819850b81dac04bb3ab9e67e0
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_24/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_25/d/dc9640b1dfd7dc75d20d072cdef305eb b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_25/d/dc9640b1dfd7dc75d20d072cdef305eb
new file mode 100644
index 0000000000000000000000000000000000000000..e4181d68d770487ab05fb67b4fa80c8e30d9aeee
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_25/d/dc9640b1dfd7dc75d20d072cdef305eb
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c08b6e14ffc4c7c50cf18e764c51588b67177b831354d43ba136fa8eaf167c4
+size 278011611
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_25/d/eeaa951f07b9a1bf11b6ae516e9ac4e0 b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_25/d/eeaa951f07b9a1bf11b6ae516e9ac4e0
new file mode 100644
index 0000000000000000000000000000000000000000..fa70dc73376af837fd9095dba68019897825abc2
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_25/d/eeaa951f07b9a1bf11b6ae516e9ac4e0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea95f4a6d1d1805598a48816b2074d506871efe6b42e4c100e0856f9a9f5c8ce
+size 62105103
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_25/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_25/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..7410019c33f4cde334ba5aa28e3227c12900bccd
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_25/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_26/d/40bd6d1cafcab5093472f08855ecb1fc b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_26/d/40bd6d1cafcab5093472f08855ecb1fc
new file mode 100644
index 0000000000000000000000000000000000000000..4ade0ecc6cf51c339e2bd1e459f2ad00a6e99e88
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_26/d/40bd6d1cafcab5093472f08855ecb1fc
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:165c0826ed0b28f67d73d99ab734a5262f19b69749d93be86832bd35ac81f682
+size 58831339
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_26/d/6081bb27ad0e0b66450d6ca947dc2658 b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_26/d/6081bb27ad0e0b66450d6ca947dc2658
new file mode 100644
index 0000000000000000000000000000000000000000..6b4f280ad9cf9f9881730f46a0fb31c7ee8f648e
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_26/d/6081bb27ad0e0b66450d6ca947dc2658
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa8847a1fd349ad0a7afa1364be0ef19950e81720936c8e2510e5039829d1fc9
+size 281075627
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_26/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_26/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..b0eb1a2c62d12bec403fcfcad5828f74ec1fab84
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_26/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_27/d/0c599c28fa763262c933067d1094605c b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_27/d/0c599c28fa763262c933067d1094605c
new file mode 100644
index 0000000000000000000000000000000000000000..0dab3c27f57ec7910bc1f03ad6215edc73269956
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_27/d/0c599c28fa763262c933067d1094605c
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3bd86f3c36dbe71e0b1d7a1cea56e94c2cc2b5f61e371a43b3e787236769370d
+size 282023889
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_27/d/f115e41d34a9d8fbfea8549bf8732ed1 b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_27/d/f115e41d34a9d8fbfea8549bf8732ed1
new file mode 100644
index 0000000000000000000000000000000000000000..e7458cf87005f7c682298e22430fa99bec222e18
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_27/d/f115e41d34a9d8fbfea8549bf8732ed1
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e86d2e31c9c110d506193b3e2f2138ff7fed0fcf8251d35bc68878715bf04f7c
+size 53542303
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_27/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_27/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..a4e371c002c6a16c936865feb9a082094ec03089
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_27/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_28/d/5f24e1bffe456424abb16367d982a77c b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_28/d/5f24e1bffe456424abb16367d982a77c
new file mode 100644
index 0000000000000000000000000000000000000000..bc9919364079b4f0fadd539e4b6e3ccc67f451df
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_28/d/5f24e1bffe456424abb16367d982a77c
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0beec9f9e3ed2c3df7b7925710a267962bb39e28ecaec870fa192f01038061e
+size 53534852
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_28/d/c6527a1b344a45832ecb7038e71abcc9 b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_28/d/c6527a1b344a45832ecb7038e71abcc9
new file mode 100644
index 0000000000000000000000000000000000000000..cb5e24cccf02824d0e5b4269ec8b1cae2c1910de
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_28/d/c6527a1b344a45832ecb7038e71abcc9
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:876dbf9a815f5060db340b5ce415ec4ac8103d92575783b9d0f7e568595a48db
+size 286756824
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_28/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_28/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..cba710e13a4c91124679c632951654fb4f81cc0c
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_28/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_29/d/79777c9a2c5a16d7704f8dc5d09b42bf b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_29/d/79777c9a2c5a16d7704f8dc5d09b42bf
new file mode 100644
index 0000000000000000000000000000000000000000..3e2ed82608c8cb2a878c0a46360261cc1c69fb72
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_29/d/79777c9a2c5a16d7704f8dc5d09b42bf
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6bbc0e631a7b1580c15753154358bc448146a7807483e04bdb89c330ab35a347
+size 286561829
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_29/d/7ac60bf8900828cfd243e9fac33458ec b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_29/d/7ac60bf8900828cfd243e9fac33458ec
new file mode 100644
index 0000000000000000000000000000000000000000..1ca806a2517f5561c745fe2e5da968e347a62737
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_29/d/7ac60bf8900828cfd243e9fac33458ec
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8cbb89c7361d25dd6e37f25c37c40423b39f97fa0aa1ec6c80b0e41adfedb615
+size 53542321
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_29/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_29/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..0e2a2e226941b62a6b50ba46ea5b4651ba43300d
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_29/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_3/d/b2c13cbf2a5a73b7e86c4335f037442b b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_3/d/b2c13cbf2a5a73b7e86c4335f037442b
new file mode 100644
index 0000000000000000000000000000000000000000..ff7ddaaf505e28afd61fd01120f3486890f96ee5
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_3/d/b2c13cbf2a5a73b7e86c4335f037442b
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d995152b4a134c8f7e73caf91f5cd31f5f35e3cac5f5ddf44b82b57e6d2acd5
+size 55724198
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_3/d/eb2fdd93d5376fa3952d28bc18e2379e b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_3/d/eb2fdd93d5376fa3952d28bc18e2379e
new file mode 100644
index 0000000000000000000000000000000000000000..4c1cc25f69fbfa8d49a2a9f9124a6a3e15e62cc7
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_3/d/eb2fdd93d5376fa3952d28bc18e2379e
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57c7d29bb6f3aaf50e08f140609d10ec916aa35bf611d48b1edadbfc2da97e5b
+size 284071941
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_3/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_3/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..b4778600b525d3f229f610f5561188c70ebc6a4d
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_3/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_30/d/5739f93aa1ce52eb1f1a3d1c12ba4d4d b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_30/d/5739f93aa1ce52eb1f1a3d1c12ba4d4d
new file mode 100644
index 0000000000000000000000000000000000000000..39b6e70dd028d4dca5db5e45573e9356d817880c
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_30/d/5739f93aa1ce52eb1f1a3d1c12ba4d4d
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:825029c76b10dce229ebb3412a671e4f9c11883873693d33ec008544a42c2b4c
+size 64320167
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_30/d/91f3fc48630d56e020a7a70cd78841fd b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_30/d/91f3fc48630d56e020a7a70cd78841fd
new file mode 100644
index 0000000000000000000000000000000000000000..d888a4c00c2a4f4e3bb42a00fda68364e94d3c55
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_30/d/91f3fc48630d56e020a7a70cd78841fd
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ebfae88d489ca882ff913b838567713d3e27ad5ca0d8179925916534a9ecc906
+size 275569995
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_30/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_30/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..4adb723ef258b9fea06dbd052e695058e73933d4
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_30/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_31/d/71162278399c9d898e1421e629bf9905 b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_31/d/71162278399c9d898e1421e629bf9905
new file mode 100644
index 0000000000000000000000000000000000000000..cdd119c6c516e4f1448794e3ddda112d7f3ef0fb
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_31/d/71162278399c9d898e1421e629bf9905
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9e3e66db9cc87f35f7f085c48f9e41d875900c36b31a679a40221b14c94ae4d
+size 275253707
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_31/d/a496131e03c3898cb7aca70bb9d40d72 b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_31/d/a496131e03c3898cb7aca70bb9d40d72
new file mode 100644
index 0000000000000000000000000000000000000000..6049a7039a685a714dc7571b2c2f6ba6ac183844
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_31/d/a496131e03c3898cb7aca70bb9d40d72
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f54f9388386365d9d44d9a9dcab3b5d4f39f263e46a9402727099b736fbcb72
+size 60104237
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_31/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_31/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..64a37a38e482287ae322ee513505543f6a654d95
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_31/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_4/d/0e2951fa0993cfe957d501beb89ae0e1 b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_4/d/0e2951fa0993cfe957d501beb89ae0e1
new file mode 100644
index 0000000000000000000000000000000000000000..0b89d1bf5ef5efaa00177e0fbfb9bb51733f0d87
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_4/d/0e2951fa0993cfe957d501beb89ae0e1
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb11bbc15337da7faf54f883ebe5c174732f799645a433d9740c008664947826
+size 62308499
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_4/d/b062f37880666b57ebdee2082f88e2c3 b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_4/d/b062f37880666b57ebdee2082f88e2c3
new file mode 100644
index 0000000000000000000000000000000000000000..531b82b2e73611afaea65e2b66c22fa156418d30
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_4/d/b062f37880666b57ebdee2082f88e2c3
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a55c1cddf001dd800b1750683044f0317a5a1c5448d592e874179f23824872f
+size 278026896
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_4/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_4/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..e7789737debd3822fd209d34198d50186cf957bc
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_4/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_5/d/aae30a0c304b761b9053580bf34b386f b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_5/d/aae30a0c304b761b9053580bf34b386f
new file mode 100644
index 0000000000000000000000000000000000000000..9c530a4c82f11b236a8945fee411e57922b019b4
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_5/d/aae30a0c304b761b9053580bf34b386f
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a73995b5afc1e951c4c1a4d9662d2b2085c41ac6f6d0fa0413c242bc73fc0a1e
+size 278617013
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_5/d/e60bd10ff8f697eab590d3d1537bde56 b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_5/d/e60bd10ff8f697eab590d3d1537bde56
new file mode 100644
index 0000000000000000000000000000000000000000..613923d9e28838c83db7034ad59fb006a00f613a
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_5/d/e60bd10ff8f697eab590d3d1537bde56
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:350a31852bce0232d702c86a5ba1018c127147c26fb2071f650675bd76c1f6cc
+size 61574545
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_5/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_5/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..3b93c7d0c50e28f41ab61f7e39b71afdab9fa516
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_5/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_6/d/7e3314a9dc17979d12616c71052d3e8b b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_6/d/7e3314a9dc17979d12616c71052d3e8b
new file mode 100644
index 0000000000000000000000000000000000000000..52f5deb6e32be8b145bf23d5cb92aefe18aee42f
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_6/d/7e3314a9dc17979d12616c71052d3e8b
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:709d50eb61dbdffdb3d0ea9ab8c7466077456e0f8e15766e86757d8800219f58
+size 273253408
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_6/d/d32f8c4ba3139b8a6a3c5ec0d4be4039 b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_6/d/d32f8c4ba3139b8a6a3c5ec0d4be4039
new file mode 100644
index 0000000000000000000000000000000000000000..5baebd51fbdb6bfe04ff79e3736b2953dfaf6e3c
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_6/d/d32f8c4ba3139b8a6a3c5ec0d4be4039
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f56f1bd717da6dad68348464a07ab2906e3779d6d8a41b8468b087798496c971
+size 66690760
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_6/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_6/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..db0c29ec6eb02fdcab9d4e69636972a1ce3a5ffb
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_6/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_7/d/6ae86240fa7e988b870ff2b4bab25afd b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_7/d/6ae86240fa7e988b870ff2b4bab25afd
new file mode 100644
index 0000000000000000000000000000000000000000..8dc56a09d27800e487d84f3175751b258ed31a10
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_7/d/6ae86240fa7e988b870ff2b4bab25afd
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af7185497c13880f977f1c2d19d9ee697ffe1487c76ebc46c37eecb5a0721ff8
+size 58830078
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_7/d/ddda27fafa957b578ae376ea148fa768 b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_7/d/ddda27fafa957b578ae376ea148fa768
new file mode 100644
index 0000000000000000000000000000000000000000..6d086a608305e0c0e5280b74323da4a588255dca
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_7/d/ddda27fafa957b578ae376ea148fa768
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6410ed76b17dfe67331156bf3bc21fb4df620c9b1693782de7dbb7ef1b413fbc
+size 280934702
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_7/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_7/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..21059e9a67776f6854f055d0c17216dff982b2e6
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_7/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_8/d/b462c404ca75a6a02e7e4fd2473c23bf b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_8/d/b462c404ca75a6a02e7e4fd2473c23bf
new file mode 100644
index 0000000000000000000000000000000000000000..32f7f94597065fad3364076fd1e7da39f76128df
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_8/d/b462c404ca75a6a02e7e4fd2473c23bf
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:adf9b5e7c2d4eea0da25c72d1e369706baaa771bad077be1204a3c46207197bb
+size 55738609
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_8/d/db63f820fe3ce3a3e4e749b70fffa79d b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_8/d/db63f820fe3ce3a3e4e749b70fffa79d
new file mode 100644
index 0000000000000000000000000000000000000000..79d1a5bfe59cfd6036484309b1bc88c1f629c6d3
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_8/d/db63f820fe3ce3a3e4e749b70fffa79d
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a3d273fef2e638f1f61221ee037d66c7d8780d4ddc04593c19632e170f0df4dd
+size 284596825
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_8/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_8/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..746017c33bda615d4d6ae0537088fe7abcf5829c
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_8/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_9/d/7c372b8786cf1c28dc59a79dcf2f7151 b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_9/d/7c372b8786cf1c28dc59a79dcf2f7151
new file mode 100644
index 0000000000000000000000000000000000000000..d6fb340d540f17e003fd8de884c942f9b0f3ac31
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_9/d/7c372b8786cf1c28dc59a79dcf2f7151
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8a104449732c32210cdd97c435e128d2d91ffa960cb90e60855309deaca0fe2
+size 284437400
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_9/d/87a425a6d7a784b99018f979e4d422f9 b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_9/d/87a425a6d7a784b99018f979e4d422f9
new file mode 100644
index 0000000000000000000000000000000000000000..973dddc668e48a13d0df33d57f23ccea15c2abbc
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_9/d/87a425a6d7a784b99018f979e4d422f9
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89585a29d7a479217abd6f39f0ac402f83a315ff66e4650228d8139f4c843b1d
+size 55732423
diff --git a/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_9/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_9/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..3726b11b26152062812e1607f155fedef52f90f4
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_model_params/ocdbt.process_9/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_model_spec/commit_success.txt b/FuSe_PaliVLA_model/15000/model_model_spec/commit_success.txt
new file mode 100644
index 0000000000000000000000000000000000000000..954c84308e4f49bb200502256aeaadb4293ce69d
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_spec/commit_success.txt
@@ -0,0 +1 @@
+Checkpoint commit was successful to gs://kyle-checkpoints-eu4/fuse-vla-checkpoints/golden-leaf-2/15000/model_model_spec
\ No newline at end of file
diff --git a/FuSe_PaliVLA_model/15000/model_model_spec/metadata b/FuSe_PaliVLA_model/15000/model_model_spec/metadata
new file mode 100644
index 0000000000000000000000000000000000000000..9b7663e0c99cd88681b564070b88cb8a7c991951
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_model_spec/metadata
@@ -0,0 +1 @@
+{"__ctor": "palivla.model.PaliVLAModel", "config": {"encoder_specs": {"digit_left": {"__ctor": "palivla.tactile_encoder_pooled.tvlViT", "config": {"num_classes": 2048, "pool_type": "cross_attn"}}, "digit_right": {"__ctor": "palivla.tactile_encoder_pooled.tvlViT", "config": {"num_classes": 2048, "pool_type": "cross_attn"}}, "modality_embedder": {"__ctor": "palivla.modality_embedder.ModalityEmbedder", "config": {"dtype_str": "float32", "embedding_dim": 2048, "num_embeddings": 9}}, "spectro": {"__ctor": "octo.model.components.vit_encoders.ResNet26FILM", "config": {"flatten_result": true, "normalize_input": false, "num_classes": 2048, "use_film": false}}}, "img_spec": {"__ctor": "big_vision.models.vit.Model", "config": {"pool_type": "none", "scan": true, "variant": "So400m/14"}}, "llm_spec": {"__ctor": "big_vision.models.proj.paligemma.gemma_bv.Model", "config": {"vocab_size": 257152}}, "modality_mappings": {"image_digit_left": "digit_left", "image_digit_right": "digit_right", "image_primary": "img", "image_wrist": "img", "mel_spectro": "spectro", "modality_idx": "modality_embedder", "text": "llm"}, "prompt_autoregressive": false, "target_key_order": ["image_primary", "image_wrist", "image_digit_left", "image_digit_right", "mel_spectro", "modality_idx"]}}
\ No newline at end of file
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/_METADATA b/FuSe_PaliVLA_model/15000/model_opt_state/_METADATA
new file mode 100644
index 0000000000000000000000000000000000000000..cbc19134316a2babf130a50c4bcaae8a79222a24
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/_METADATA
@@ -0,0 +1 @@
+{"tree_metadata": {"('opt_state', 'count')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "count", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'hyperparams', 'embed_learning_rate')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "hyperparams", "key_type": 2}, {"key": "embed_learning_rate", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'hyperparams', 'img_learning_rate')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "hyperparams", "key_type": 2}, {"key": "img_learning_rate", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'hyperparams', 'llm_learning_rate')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "hyperparams", "key_type": 2}, {"key": "llm_learning_rate", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'hyperparams_states', 'embed_learning_rate', 'count')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "hyperparams_states", "key_type": 2}, {"key": "embed_learning_rate", "key_type": 2}, {"key": "count", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'hyperparams_states', 'img_learning_rate', 'count')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "hyperparams_states", "key_type": 2}, {"key": "img_learning_rate", "key_type": 2}, {"key": "count", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'hyperparams_states', 'llm_learning_rate', 'count')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "hyperparams_states", "key_type": 2}, {"key": "llm_learning_rate", "key_type": 2}, {"key": "count", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '0')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "0", "key_type": 1}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'count')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "count", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block0', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block0', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block0', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block0', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block0', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block0', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block0', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block0', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block0', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block0', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block0', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block0', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block1', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block1', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block1', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block1', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block1', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block1', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block1', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block1', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block1', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block1', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block1', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block1', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block10', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block10', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block10', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block10', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block10', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block10', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block10', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block10', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block10', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block10', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block10', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block10', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block11', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block11', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block11', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block11', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block11', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block11', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block11', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block11', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block11', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block11', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block11', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block11', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block2', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block2', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block2', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block2', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block2', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block2', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block2', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block2', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block2', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block2', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block2', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block2', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block3', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block3', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block3', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block3', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block3', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block3', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block3', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block3', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block3', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block3', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block3', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block3', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block4', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block4', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block4', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block4', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block4', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block4', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block4', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block4', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block4', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block4', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block4', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block4', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block5', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block5', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block5', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block5', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block5', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block5', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block5', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block5', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block5', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block5', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block5', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block5', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block6', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block6', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block6', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block6', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block6', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block6', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block6', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block6', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block6', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block6', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block6', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block6', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block7', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block7', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block7', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block7', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block7', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block7', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block7', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block7', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block7', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block7', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block7', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block7', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block8', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block8', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block8', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block8', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block8', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block8', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block8', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block8', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block8', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block8', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block8', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block8', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block9', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block9', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block9', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block9', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block9', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block9', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block9', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block9', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block9', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block9', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block9', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'block9', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'cls_token')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "cls_token", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'fc_norm', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "fc_norm", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'fc_norm', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "fc_norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'head', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "head", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'head', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "head", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'patch_embed', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "patch_embed", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'patch_embed', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "patch_embed", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'pool_func', 'attention_head', 'key', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "key", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'pool_func', 'attention_head', 'key', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "key", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'pool_func', 'attention_head', 'out', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "out", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'pool_func', 'attention_head', 'out', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "out", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'pool_func', 'attention_head', 'query', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "query", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'pool_func', 'attention_head', 'query', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "query", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'pool_func', 'attention_head', 'value', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "value", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'pool_func', 'attention_head', 'value', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "value", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'pool_func', 'readouts')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "readouts", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_left', 'pos_embed')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pos_embed", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block0', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block0', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block0', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block0', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block0', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block0', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block0', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block0', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block0', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block0', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block0', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block0', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block1', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block1', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block1', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block1', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block1', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block1', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block1', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block1', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block1', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block1', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block1', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block1', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block10', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block10', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block10', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block10', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block10', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block10', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block10', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block10', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block10', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block10', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block10', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block10', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block11', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block11', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block11', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block11', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block11', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block11', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block11', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block11', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block11', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block11', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block11', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block11', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block2', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block2', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block2', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block2', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block2', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block2', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block2', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block2', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block2', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block2', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block2', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block2', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block3', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block3', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block3', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block3', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block3', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block3', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block3', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block3', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block3', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block3', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block3', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block3', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block4', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block4', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block4', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block4', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block4', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block4', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block4', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block4', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block4', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block4', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block4', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block4', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block5', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block5', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block5', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block5', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block5', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block5', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block5', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block5', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block5', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block5', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block5', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block5', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block6', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block6', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block6', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block6', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block6', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block6', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block6', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block6', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block6', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block6', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block6', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block6', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block7', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block7', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block7', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block7', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block7', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block7', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block7', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block7', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block7', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block7', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block7', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block7', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block8', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block8', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block8', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block8', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block8', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block8', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block8', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block8', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block8', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block8', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block8', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block8', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block9', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block9', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block9', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block9', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block9', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block9', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block9', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block9', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block9', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block9', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block9', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'block9', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'cls_token')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "cls_token", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'fc_norm', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "fc_norm", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'fc_norm', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "fc_norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'head', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "head", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'head', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "head", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'patch_embed', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "patch_embed", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'patch_embed', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "patch_embed", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'pool_func', 'attention_head', 'key', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "key", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'pool_func', 'attention_head', 'key', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "key", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'pool_func', 'attention_head', 'out', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "out", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'pool_func', 'attention_head', 'out', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "out", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'pool_func', 'attention_head', 'query', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "query", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'pool_func', 'attention_head', 'query', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "query", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'pool_func', 'attention_head', 'value', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "value", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'pool_func', 'attention_head', 'value', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "value", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'pool_func', 'readouts')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "readouts", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'digit_right', 'pos_embed')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pos_embed", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoder_norm', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoder_norm", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoder_norm', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoder_norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'LayerNorm_0', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "LayerNorm_0", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'LayerNorm_0', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "LayerNorm_0", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'LayerNorm_1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "LayerNorm_1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'LayerNorm_1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "LayerNorm_1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'MlpBlock_0', 'Dense_0', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MlpBlock_0", "key_type": 2}, {"key": "Dense_0", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'MlpBlock_0', 'Dense_0', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MlpBlock_0", "key_type": 2}, {"key": "Dense_0", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'MlpBlock_0', 'Dense_1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MlpBlock_0", "key_type": 2}, {"key": "Dense_1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'MlpBlock_0', 'Dense_1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MlpBlock_0", "key_type": 2}, {"key": "Dense_1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'key', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "key", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'key', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "key", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'out', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "out", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'out', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "out", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'query', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "query", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'query', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "query", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'value', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "value", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'value', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "value", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'img', 'embedding', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "embedding", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'img', 'embedding', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "embedding", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'img', 'head', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "head", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'img', 'head', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "head", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'img', 'pos_embedding')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "pos_embedding", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'llm', 'embedder', 'input_embedding')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "embedder", "key_type": 2}, {"key": "input_embedding", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'llm', 'final_norm', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "final_norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'llm', 'layers', 'attn', 'attn_vec_einsum', 'w')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "attn_vec_einsum", "key_type": 2}, {"key": "w", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'llm', 'layers', 'attn', 'kv_einsum', 'w')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "kv_einsum", "key_type": 2}, {"key": "w", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'llm', 'layers', 'attn', 'q_einsum', 'w')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "q_einsum", "key_type": 2}, {"key": "w", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'llm', 'layers', 'mlp', 'gating_einsum')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "gating_einsum", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'llm', 'layers', 'mlp', 'linear')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "linear", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'llm', 'layers', 'pre_attention_norm', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "pre_attention_norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'llm', 'layers', 'pre_ffw_norm', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "pre_ffw_norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'modality_embedder', 'Embed_0', 'embedding')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "modality_embedder", "key_type": 2}, {"key": "Embed_0", "key_type": 2}, {"key": "embedding", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'Dense_0', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "Dense_0", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'Dense_0', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "Dense_0", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit1', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit1', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit1', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit1', 'conv_proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit1', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit1', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit1', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit1', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit1', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit1', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit1', 'gn_proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit1', 'gn_proj', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit2', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit2', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit2', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit2', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit2', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit2', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit2', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit2', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit2', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit1', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit1', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit1', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit1', 'conv_proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit1', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit1', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit1', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit1', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit1', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit1', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit1', 'gn_proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit1', 'gn_proj', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit2', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit2', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit2', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit2', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit2', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit2', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit2', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit2', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit2', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit1', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit1', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit1', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit1', 'conv_proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit1', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit1', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit1', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit1', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit1', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit1', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit1', 'gn_proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit1', 'gn_proj', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit2', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit2', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit2', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit2', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit2', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit2', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit2', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit2', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit2', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit1', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit1', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit1', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit1', 'conv_proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit1', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit1', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit1', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit1', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit1', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit1', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit1', 'gn_proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit1', 'gn_proj', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit2', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit2', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit2', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit2', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit2', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit2', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit2', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit2', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit2', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'conv_root', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "conv_root", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'gn_root', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "gn_root", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'spectro', 'gn_root', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "gn_root", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'start_image_digit_left')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "start_image_digit_left", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'start_image_digit_right')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "start_image_digit_right", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'start_image_primary')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "start_image_primary", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'start_image_wrist')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "start_image_wrist", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'start_mel_spectro')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "start_mel_spectro", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'start_modality_idx')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "start_modality_idx", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'mu', 'start_text')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "start_text", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block0', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block0', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block0', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block0', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block0', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block0', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block0', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block0', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block0', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block0', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block0', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block0', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block1', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block1', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block1', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block1', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block1', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block1', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block1', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block1', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block1', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block1', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block1', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block1', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block10', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block10', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block10', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block10', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block10', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block10', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block10', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block10', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block10', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block10', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block10', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block10', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block11', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block11', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block11', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block11', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block11', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block11', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block11', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block11', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block11', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block11', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block11', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block11', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block2', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block2', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block2', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block2', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block2', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block2', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block2', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block2', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block2', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block2', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block2', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block2', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block3', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block3', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block3', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block3', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block3', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block3', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block3', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block3', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block3', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block3', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block3', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block3', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block4', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block4', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block4', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block4', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block4', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block4', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block4', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block4', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block4', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block4', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block4', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block4', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block5', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block5', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block5', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block5', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block5', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block5', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block5', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block5', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block5', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block5', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block5', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block5', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block6', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block6', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block6', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block6', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block6', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block6', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block6', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block6', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block6', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block6', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block6', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block6', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block7', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block7', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block7', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block7', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block7', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block7', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block7', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block7', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block7', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block7', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block7', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block7', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block8', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block8', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block8', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block8', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block8', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block8', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block8', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block8', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block8', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block8', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block8', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block8', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block9', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block9', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block9', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block9', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block9', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block9', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block9', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block9', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block9', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block9', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block9', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'block9', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'cls_token')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "cls_token", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'fc_norm', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "fc_norm", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'fc_norm', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "fc_norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'head', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "head", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'head', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "head", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'patch_embed', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "patch_embed", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'patch_embed', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "patch_embed", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'pool_func', 'attention_head', 'key', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "key", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'pool_func', 'attention_head', 'key', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "key", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'pool_func', 'attention_head', 'out', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "out", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'pool_func', 'attention_head', 'out', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "out", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'pool_func', 'attention_head', 'query', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "query", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'pool_func', 'attention_head', 'query', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "query", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'pool_func', 'attention_head', 'value', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "value", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'pool_func', 'attention_head', 'value', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "value", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'pool_func', 'readouts')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "readouts", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_left', 'pos_embed')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pos_embed", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block0', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block0', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block0', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block0', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block0', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block0', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block0', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block0', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block0', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block0', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block0', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block0', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block1', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block1', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block1', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block1', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block1', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block1', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block1', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block1', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block1', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block1', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block1', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block1', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block10', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block10', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block10', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block10', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block10', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block10', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block10', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block10', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block10', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block10', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block10', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block10', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block11', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block11', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block11', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block11', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block11', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block11', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block11', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block11', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block11', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block11', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block11', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block11', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block2', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block2', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block2', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block2', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block2', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block2', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block2', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block2', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block2', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block2', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block2', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block2', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block3', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block3', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block3', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block3', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block3', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block3', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block3', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block3', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block3', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block3', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block3', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block3', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block4', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block4', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block4', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block4', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block4', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block4', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block4', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block4', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block4', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block4', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block4', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block4', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block5', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block5', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block5', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block5', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block5', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block5', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block5', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block5', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block5', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block5', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block5', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block5', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block6', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block6', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block6', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block6', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block6', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block6', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block6', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block6', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block6', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block6', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block6', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block6', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block7', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block7', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block7', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block7', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block7', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block7', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block7', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block7', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block7', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block7', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block7', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block7', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block8', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block8', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block8', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block8', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block8', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block8', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block8', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block8', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block8', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block8', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block8', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block8', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block9', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block9', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block9', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block9', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block9', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block9', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block9', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block9', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block9', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block9', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block9', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'block9', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'cls_token')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "cls_token", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'fc_norm', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "fc_norm", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'fc_norm', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "fc_norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'head', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "head", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'head', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "head", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'patch_embed', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "patch_embed", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'patch_embed', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "patch_embed", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'pool_func', 'attention_head', 'key', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "key", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'pool_func', 'attention_head', 'key', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "key", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'pool_func', 'attention_head', 'out', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "out", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'pool_func', 'attention_head', 'out', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "out", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'pool_func', 'attention_head', 'query', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "query", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'pool_func', 'attention_head', 'query', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "query", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'pool_func', 'attention_head', 'value', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "value", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'pool_func', 'attention_head', 'value', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "value", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'pool_func', 'readouts')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "readouts", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'digit_right', 'pos_embed')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pos_embed", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoder_norm', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoder_norm", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoder_norm', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoder_norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'LayerNorm_0', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "LayerNorm_0", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'LayerNorm_0', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "LayerNorm_0", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'LayerNorm_1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "LayerNorm_1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'LayerNorm_1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "LayerNorm_1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'MlpBlock_0', 'Dense_0', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MlpBlock_0", "key_type": 2}, {"key": "Dense_0", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'MlpBlock_0', 'Dense_0', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MlpBlock_0", "key_type": 2}, {"key": "Dense_0", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'MlpBlock_0', 'Dense_1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MlpBlock_0", "key_type": 2}, {"key": "Dense_1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'MlpBlock_0', 'Dense_1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MlpBlock_0", "key_type": 2}, {"key": "Dense_1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'key', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "key", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'key', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "key", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'out', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "out", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'out', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "out", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'query', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "query", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'query', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "query", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'value', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "value", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'value', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "value", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'img', 'embedding', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "embedding", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'img', 'embedding', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "embedding", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'img', 'head', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "head", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'img', 'head', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "head", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'img', 'pos_embedding')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "pos_embedding", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'llm', 'embedder', 'input_embedding')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "embedder", "key_type": 2}, {"key": "input_embedding", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'llm', 'final_norm', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "final_norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'llm', 'layers', 'attn', 'attn_vec_einsum', 'w')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "attn_vec_einsum", "key_type": 2}, {"key": "w", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'llm', 'layers', 'attn', 'kv_einsum', 'w')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "kv_einsum", "key_type": 2}, {"key": "w", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'llm', 'layers', 'attn', 'q_einsum', 'w')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "q_einsum", "key_type": 2}, {"key": "w", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'llm', 'layers', 'mlp', 'gating_einsum')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "gating_einsum", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'llm', 'layers', 'mlp', 'linear')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "linear", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'llm', 'layers', 'pre_attention_norm', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "pre_attention_norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'llm', 'layers', 'pre_ffw_norm', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "pre_ffw_norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'modality_embedder', 'Embed_0', 'embedding')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "modality_embedder", "key_type": 2}, {"key": "Embed_0", "key_type": 2}, {"key": "embedding", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'Dense_0', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "Dense_0", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'Dense_0', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "Dense_0", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit1', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit1', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit1', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit1', 'conv_proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit1', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit1', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit1', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit1', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit1', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit1', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit1', 'gn_proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit1', 'gn_proj', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit2', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit2', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit2', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit2', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit2', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit2', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit2', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit2', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit2', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit1', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit1', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit1', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit1', 'conv_proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit1', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit1', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit1', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit1', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit1', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit1', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit1', 'gn_proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit1', 'gn_proj', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit2', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit2', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit2', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit2', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit2', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit2', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit2', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit2', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit2', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit1', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit1', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit1', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit1', 'conv_proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit1', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit1', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit1', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit1', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit1', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit1', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit1', 'gn_proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit1', 'gn_proj', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit2', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit2', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit2', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit2', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit2', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit2', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit2', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit2', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit2', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit1', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit1', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit1', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit1', 'conv_proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit1', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit1', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit1', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit1', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit1', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit1', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit1', 'gn_proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit1', 'gn_proj', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit2', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit2', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit2', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit2', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit2', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit2', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit2', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit2', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit2', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'conv_root', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "conv_root", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'gn_root', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "gn_root", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'spectro', 'gn_root', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "gn_root", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'start_image_digit_left')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "start_image_digit_left", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'start_image_digit_right')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "start_image_digit_right", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'start_image_primary')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "start_image_primary", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'start_image_wrist')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "start_image_wrist", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'start_mel_spectro')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "start_mel_spectro", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'start_modality_idx')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "start_modality_idx", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '0', 'nu', 'start_text')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "start_text", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '1')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "1", "key_type": 1}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'embed', 'inner_state', '1', '2')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "embed", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "2", "key_type": 1}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '0')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "0", "key_type": 1}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'count')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "count", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block0', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block0', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block0', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block0', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block0', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block0', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block0', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block0', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block0', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block0', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block0', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block0', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block1', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block1', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block1', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block1', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block1', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block1', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block1', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block1', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block1', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block1', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block1', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block1', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block10', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block10', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block10', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block10', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block10', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block10', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block10', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block10', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block10', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block10', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block10', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block10', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block11', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block11', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block11', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block11', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block11', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block11', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block11', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block11', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block11', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block11', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block11', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block11', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block2', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block2', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block2', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block2', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block2', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block2', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block2', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block2', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block2', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block2', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block2', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block2', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block3', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block3', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block3', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block3', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block3', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block3', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block3', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block3', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block3', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block3', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block3', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block3', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block4', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block4', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block4', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block4', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block4', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block4', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block4', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block4', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block4', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block4', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block4', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block4', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block5', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block5', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block5', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block5', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block5', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block5', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block5', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block5', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block5', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block5', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block5', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block5', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block6', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block6', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block6', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block6', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block6', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block6', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block6', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block6', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block6', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block6', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block6', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block6', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block7', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block7', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block7', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block7', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block7', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block7', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block7', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block7', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block7', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block7', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block7', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block7', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block8', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block8', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block8', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block8', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block8', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block8', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block8', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block8', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block8', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block8', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block8', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block8', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block9', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block9', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block9', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block9', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block9', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block9', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block9', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block9', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block9', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block9', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block9', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'block9', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'cls_token')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "cls_token", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'fc_norm', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "fc_norm", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'fc_norm', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "fc_norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'head', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "head", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'head', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "head", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'patch_embed', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "patch_embed", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'patch_embed', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "patch_embed", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'pool_func', 'attention_head', 'key', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "key", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'pool_func', 'attention_head', 'key', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "key", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'pool_func', 'attention_head', 'out', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "out", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'pool_func', 'attention_head', 'out', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "out", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'pool_func', 'attention_head', 'query', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "query", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'pool_func', 'attention_head', 'query', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "query", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'pool_func', 'attention_head', 'value', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "value", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'pool_func', 'attention_head', 'value', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "value", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'pool_func', 'readouts')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "readouts", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_left', 'pos_embed')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pos_embed", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block0', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block0', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block0', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block0', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block0', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block0', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block0', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block0', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block0', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block0', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block0', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block0', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block1', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block1', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block1', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block1', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block1', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block1', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block1', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block1', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block1', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block1', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block1', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block1', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block10', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block10', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block10', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block10', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block10', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block10', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block10', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block10', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block10', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block10', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block10', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block10', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block11', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block11', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block11', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block11', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block11', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block11', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block11', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block11', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block11', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block11', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block11', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block11', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block2', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block2', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block2', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block2', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block2', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block2', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block2', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block2', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block2', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block2', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block2', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block2', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block3', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block3', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block3', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block3', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block3', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block3', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block3', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block3', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block3', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block3', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block3', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block3', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block4', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block4', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block4', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block4', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block4', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block4', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block4', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block4', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block4', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block4', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block4', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block4', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block5', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block5', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block5', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block5', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block5', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block5', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block5', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block5', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block5', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block5', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block5', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block5', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block6', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block6', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block6', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block6', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block6', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block6', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block6', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block6', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block6', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block6', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block6', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block6', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block7', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block7', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block7', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block7', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block7', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block7', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block7', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block7', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block7', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block7', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block7', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block7', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block8', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block8', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block8', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block8', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block8', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block8', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block8', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block8', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block8', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block8', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block8', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block8', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block9', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block9', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block9', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block9', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block9', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block9', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block9', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block9', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block9', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block9', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block9', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'block9', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'cls_token')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "cls_token", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'fc_norm', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "fc_norm", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'fc_norm', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "fc_norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'head', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "head", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'head', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "head", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'patch_embed', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "patch_embed", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'patch_embed', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "patch_embed", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'pool_func', 'attention_head', 'key', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "key", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'pool_func', 'attention_head', 'key', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "key", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'pool_func', 'attention_head', 'out', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "out", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'pool_func', 'attention_head', 'out', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "out", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'pool_func', 'attention_head', 'query', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "query", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'pool_func', 'attention_head', 'query', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "query", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'pool_func', 'attention_head', 'value', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "value", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'pool_func', 'attention_head', 'value', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "value", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'pool_func', 'readouts')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "readouts", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'digit_right', 'pos_embed')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pos_embed", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoder_norm', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoder_norm", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoder_norm', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoder_norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'LayerNorm_0', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "LayerNorm_0", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'LayerNorm_0', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "LayerNorm_0", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'LayerNorm_1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "LayerNorm_1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'LayerNorm_1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "LayerNorm_1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'MlpBlock_0', 'Dense_0', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MlpBlock_0", "key_type": 2}, {"key": "Dense_0", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'MlpBlock_0', 'Dense_0', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MlpBlock_0", "key_type": 2}, {"key": "Dense_0", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'MlpBlock_0', 'Dense_1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MlpBlock_0", "key_type": 2}, {"key": "Dense_1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'MlpBlock_0', 'Dense_1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MlpBlock_0", "key_type": 2}, {"key": "Dense_1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'key', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "key", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'key', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "key", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'out', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "out", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'out', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "out", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'query', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "query", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'query', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "query", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'value', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "value", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'value', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "value", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'img', 'embedding', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "embedding", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'img', 'embedding', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "embedding", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'img', 'head', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "head", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'img', 'head', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "head", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'img', 'pos_embedding')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "pos_embedding", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'llm', 'embedder', 'input_embedding')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "embedder", "key_type": 2}, {"key": "input_embedding", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'llm', 'final_norm', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "final_norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'llm', 'layers', 'attn', 'attn_vec_einsum', 'w')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "attn_vec_einsum", "key_type": 2}, {"key": "w", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'llm', 'layers', 'attn', 'kv_einsum', 'w')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "kv_einsum", "key_type": 2}, {"key": "w", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'llm', 'layers', 'attn', 'q_einsum', 'w')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "q_einsum", "key_type": 2}, {"key": "w", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'llm', 'layers', 'mlp', 'gating_einsum')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "gating_einsum", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'llm', 'layers', 'mlp', 'linear')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "linear", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'llm', 'layers', 'pre_attention_norm', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "pre_attention_norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'llm', 'layers', 'pre_ffw_norm', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "pre_ffw_norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'modality_embedder', 'Embed_0', 'embedding')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "modality_embedder", "key_type": 2}, {"key": "Embed_0", "key_type": 2}, {"key": "embedding", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'Dense_0', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "Dense_0", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'Dense_0', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "Dense_0", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit1', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit1', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit1', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit1', 'conv_proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit1', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit1', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit1', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit1', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit1', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit1', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit1', 'gn_proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit1', 'gn_proj', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit2', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit2', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit2', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit2', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit2', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit2', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit2', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit2', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit2', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit1', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit1', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit1', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit1', 'conv_proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit1', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit1', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit1', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit1', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit1', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit1', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit1', 'gn_proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit1', 'gn_proj', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit2', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit2', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit2', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit2', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit2', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit2', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit2', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit2', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit2', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit1', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit1', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit1', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit1', 'conv_proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit1', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit1', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit1', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit1', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit1', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit1', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit1', 'gn_proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit1', 'gn_proj', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit2', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit2', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit2', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit2', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit2', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit2', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit2', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit2', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit2', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit1', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit1', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit1', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit1', 'conv_proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit1', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit1', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit1', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit1', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit1', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit1', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit1', 'gn_proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit1', 'gn_proj', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit2', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit2', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit2', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit2', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit2', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit2', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit2', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit2', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit2', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'conv_root', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "conv_root", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'gn_root', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "gn_root", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'spectro', 'gn_root', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "gn_root", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'start_image_digit_left')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "start_image_digit_left", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'start_image_digit_right')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "start_image_digit_right", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'start_image_primary')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "start_image_primary", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'start_image_wrist')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "start_image_wrist", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'start_mel_spectro')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "start_mel_spectro", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'start_modality_idx')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "start_modality_idx", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'mu', 'start_text')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "start_text", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block0', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block0', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block0', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block0', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block0', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block0', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block0', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block0', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block0', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block0', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block0', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block0', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block1', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block1', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block1', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block1', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block1', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block1', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block1', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block1', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block1', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block1', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block1', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block1', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block10', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block10', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block10', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block10', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block10', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block10', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block10', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block10', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block10', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block10', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block10', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block10', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block11', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block11', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block11', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block11', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block11', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block11', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block11', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block11', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block11', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block11', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block11', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block11', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block2', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block2', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block2', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block2', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block2', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block2', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block2', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block2', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block2', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block2', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block2', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block2', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block3', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block3', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block3', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block3', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block3', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block3', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block3', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block3', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block3', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block3', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block3', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block3', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block4', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block4', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block4', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block4', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block4', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block4', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block4', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block4', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block4', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block4', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block4', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block4', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block5', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block5', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block5', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block5', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block5', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block5', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block5', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block5', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block5', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block5', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block5', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block5', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block6', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block6', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block6', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block6', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block6', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block6', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block6', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block6', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block6', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block6', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block6', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block6', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block7', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block7', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block7', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block7', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block7', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block7', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block7', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block7', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block7', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block7', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block7', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block7', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block8', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block8', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block8', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block8', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block8', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block8', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block8', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block8', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block8', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block8', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block8', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block8', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block9', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block9', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block9', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block9', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block9', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block9', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block9', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block9', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block9', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block9', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block9', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'block9', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'cls_token')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "cls_token", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'fc_norm', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "fc_norm", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'fc_norm', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "fc_norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'head', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "head", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'head', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "head", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'patch_embed', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "patch_embed", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'patch_embed', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "patch_embed", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'pool_func', 'attention_head', 'key', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "key", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'pool_func', 'attention_head', 'key', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "key", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'pool_func', 'attention_head', 'out', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "out", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'pool_func', 'attention_head', 'out', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "out", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'pool_func', 'attention_head', 'query', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "query", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'pool_func', 'attention_head', 'query', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "query", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'pool_func', 'attention_head', 'value', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "value", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'pool_func', 'attention_head', 'value', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "value", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'pool_func', 'readouts')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "readouts", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_left', 'pos_embed')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pos_embed", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block0', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block0', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block0', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block0', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block0', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block0', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block0', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block0', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block0', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block0', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block0', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block0', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block1', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block1', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block1', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block1', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block1', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block1', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block1', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block1', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block1', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block1', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block1', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block1', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block10', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block10', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block10', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block10', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block10', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block10', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block10', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block10', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block10', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block10', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block10', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block10', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block11', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block11', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block11', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block11', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block11', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block11', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block11', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block11', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block11', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block11', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block11', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block11', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block2', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block2', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block2', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block2', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block2', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block2', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block2', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block2', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block2', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block2', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block2', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block2', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block3', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block3', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block3', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block3', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block3', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block3', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block3', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block3', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block3', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block3', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block3', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block3', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block4', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block4', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block4', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block4', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block4', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block4', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block4', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block4', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block4', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block4', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block4', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block4', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block5', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block5', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block5', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block5', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block5', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block5', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block5', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block5', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block5', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block5', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block5', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block5', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block6', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block6', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block6', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block6', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block6', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block6', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block6', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block6', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block6', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block6', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block6', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block6', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block7', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block7', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block7', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block7', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block7', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block7', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block7', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block7', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block7', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block7', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block7', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block7', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block8', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block8', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block8', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block8', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block8', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block8', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block8', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block8', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block8', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block8', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block8', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block8', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block9', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block9', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block9', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block9', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block9', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block9', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block9', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block9', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block9', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block9', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block9', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'block9', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'cls_token')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "cls_token", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'fc_norm', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "fc_norm", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'fc_norm', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "fc_norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'head', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "head", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'head', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "head", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'patch_embed', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "patch_embed", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'patch_embed', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "patch_embed", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'pool_func', 'attention_head', 'key', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "key", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'pool_func', 'attention_head', 'key', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "key", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'pool_func', 'attention_head', 'out', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "out", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'pool_func', 'attention_head', 'out', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "out", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'pool_func', 'attention_head', 'query', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "query", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'pool_func', 'attention_head', 'query', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "query", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'pool_func', 'attention_head', 'value', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "value", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'pool_func', 'attention_head', 'value', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "value", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'pool_func', 'readouts')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "readouts", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'digit_right', 'pos_embed')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pos_embed", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoder_norm', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoder_norm", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoder_norm', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoder_norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'LayerNorm_0', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "LayerNorm_0", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'LayerNorm_0', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "LayerNorm_0", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'LayerNorm_1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "LayerNorm_1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'LayerNorm_1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "LayerNorm_1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'MlpBlock_0', 'Dense_0', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MlpBlock_0", "key_type": 2}, {"key": "Dense_0", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'MlpBlock_0', 'Dense_0', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MlpBlock_0", "key_type": 2}, {"key": "Dense_0", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'MlpBlock_0', 'Dense_1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MlpBlock_0", "key_type": 2}, {"key": "Dense_1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'MlpBlock_0', 'Dense_1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MlpBlock_0", "key_type": 2}, {"key": "Dense_1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'key', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "key", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'key', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "key", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'out', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "out", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'out', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "out", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'query', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "query", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'query', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "query", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'value', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "value", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'value', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "value", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'img', 'embedding', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "embedding", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'img', 'embedding', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "embedding", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'img', 'head', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "head", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'img', 'head', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "head", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'img', 'pos_embedding')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "pos_embedding", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'llm', 'embedder', 'input_embedding')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "embedder", "key_type": 2}, {"key": "input_embedding", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'llm', 'final_norm', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "final_norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'llm', 'layers', 'attn', 'attn_vec_einsum', 'w')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "attn_vec_einsum", "key_type": 2}, {"key": "w", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'llm', 'layers', 'attn', 'kv_einsum', 'w')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "kv_einsum", "key_type": 2}, {"key": "w", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'llm', 'layers', 'attn', 'q_einsum', 'w')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "q_einsum", "key_type": 2}, {"key": "w", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'llm', 'layers', 'mlp', 'gating_einsum')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "gating_einsum", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'llm', 'layers', 'mlp', 'linear')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "linear", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'llm', 'layers', 'pre_attention_norm', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "pre_attention_norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'llm', 'layers', 'pre_ffw_norm', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "pre_ffw_norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'modality_embedder', 'Embed_0', 'embedding')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "modality_embedder", "key_type": 2}, {"key": "Embed_0", "key_type": 2}, {"key": "embedding", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'Dense_0', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "Dense_0", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'Dense_0', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "Dense_0", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit1', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit1', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit1', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit1', 'conv_proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit1', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit1', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit1', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit1', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit1', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit1', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit1', 'gn_proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit1', 'gn_proj', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit2', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit2', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit2', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit2', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit2', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit2', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit2', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit2', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit2', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit1', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit1', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit1', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit1', 'conv_proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit1', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit1', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit1', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit1', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit1', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit1', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit1', 'gn_proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit1', 'gn_proj', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit2', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit2', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit2', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit2', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit2', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit2', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit2', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit2', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit2', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit1', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit1', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit1', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit1', 'conv_proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit1', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit1', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit1', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit1', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit1', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit1', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit1', 'gn_proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit1', 'gn_proj', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit2', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit2', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit2', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit2', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit2', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit2', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit2', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit2', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit2', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit1', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit1', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit1', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit1', 'conv_proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit1', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit1', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit1', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit1', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit1', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit1', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit1', 'gn_proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit1', 'gn_proj', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit2', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit2', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit2', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit2', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit2', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit2', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit2', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit2', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit2', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'conv_root', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "conv_root", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'gn_root', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "gn_root", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'spectro', 'gn_root', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "gn_root", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'start_image_digit_left')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "start_image_digit_left", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'start_image_digit_right')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "start_image_digit_right", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'start_image_primary')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "start_image_primary", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'start_image_wrist')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "start_image_wrist", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'start_mel_spectro')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "start_mel_spectro", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'start_modality_idx')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "start_modality_idx", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '0', 'nu', 'start_text')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "start_text", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '1')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "1", "key_type": 1}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'img', 'inner_state', '1', '2')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "2", "key_type": 1}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '0')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "0", "key_type": 1}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'count')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "count", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block0', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block0', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block0', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block0', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block0', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block0', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block0', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block0', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block0', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block0', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block0', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block0', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block1', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block1', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block1', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block1', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block1', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block1', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block1', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block1', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block1', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block1', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block1', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block1', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block10', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block10', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block10', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block10', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block10', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block10', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block10', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block10', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block10', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block10', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block10', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block10', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block11', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block11', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block11', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block11', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block11', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block11', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block11', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block11', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block11', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block11', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block11', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block11', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block2', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block2', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block2', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block2', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block2', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block2', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block2', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block2', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block2', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block2', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block2', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block2', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block3', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block3', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block3', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block3', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block3', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block3', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block3', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block3', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block3', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block3', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block3', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block3', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block4', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block4', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block4', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block4', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block4', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block4', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block4', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block4', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block4', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block4', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block4', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block4', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block5', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block5', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block5', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block5', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block5', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block5', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block5', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block5', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block5', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block5', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block5', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block5', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block6', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block6', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block6', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block6', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block6', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block6', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block6', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block6', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block6', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block6', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block6', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block6', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block7', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block7', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block7', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block7', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block7', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block7', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block7', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block7', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block7', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block7', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block7', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block7', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block8', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block8', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block8', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block8', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block8', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block8', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block8', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block8', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block8', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block8', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block8', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block8', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block9', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block9', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block9', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block9', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block9', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block9', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block9', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block9', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block9', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block9', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block9', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'block9', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'cls_token')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "cls_token", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'fc_norm', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "fc_norm", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'fc_norm', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "fc_norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'head', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "head", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'head', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "head", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'patch_embed', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "patch_embed", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'patch_embed', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "patch_embed", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'pool_func', 'attention_head', 'key', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "key", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'pool_func', 'attention_head', 'key', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "key", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'pool_func', 'attention_head', 'out', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "out", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'pool_func', 'attention_head', 'out', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "out", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'pool_func', 'attention_head', 'query', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "query", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'pool_func', 'attention_head', 'query', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "query", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'pool_func', 'attention_head', 'value', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "value", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'pool_func', 'attention_head', 'value', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "value", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'pool_func', 'readouts')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "readouts", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_left', 'pos_embed')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pos_embed", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block0', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block0', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block0', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block0', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block0', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block0', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block0', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block0', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block0', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block0', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block0', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block0', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block1', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block1', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block1', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block1', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block1', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block1', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block1', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block1', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block1', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block1', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block1', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block1', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block10', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block10', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block10', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block10', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block10', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block10', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block10', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block10', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block10', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block10', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block10', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block10', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block11', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block11', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block11', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block11', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block11', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block11', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block11', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block11', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block11', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block11', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block11', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block11', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block2', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block2', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block2', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block2', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block2', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block2', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block2', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block2', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block2', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block2', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block2', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block2', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block3', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block3', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block3', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block3', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block3', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block3', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block3', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block3', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block3', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block3', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block3', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block3', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block4', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block4', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block4', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block4', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block4', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block4', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block4', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block4', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block4', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block4', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block4', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block4', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block5', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block5', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block5', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block5', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block5', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block5', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block5', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block5', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block5', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block5', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block5', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block5', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block6', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block6', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block6', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block6', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block6', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block6', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block6', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block6', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block6', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block6', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block6', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block6', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block7', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block7', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block7', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block7', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block7', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block7', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block7', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block7', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block7', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block7', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block7', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block7', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block8', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block8', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block8', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block8', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block8', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block8', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block8', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block8', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block8', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block8', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block8', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block8', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block9', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block9', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block9', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block9', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block9', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block9', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block9', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block9', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block9', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block9', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block9', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'block9', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'cls_token')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "cls_token", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'fc_norm', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "fc_norm", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'fc_norm', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "fc_norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'head', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "head", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'head', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "head", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'patch_embed', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "patch_embed", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'patch_embed', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "patch_embed", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'pool_func', 'attention_head', 'key', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "key", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'pool_func', 'attention_head', 'key', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "key", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'pool_func', 'attention_head', 'out', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "out", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'pool_func', 'attention_head', 'out', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "out", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'pool_func', 'attention_head', 'query', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "query", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'pool_func', 'attention_head', 'query', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "query", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'pool_func', 'attention_head', 'value', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "value", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'pool_func', 'attention_head', 'value', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "value", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'pool_func', 'readouts')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "readouts", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'digit_right', 'pos_embed')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pos_embed", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoder_norm', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoder_norm", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoder_norm', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoder_norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'LayerNorm_0', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "LayerNorm_0", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'LayerNorm_0', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "LayerNorm_0", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'LayerNorm_1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "LayerNorm_1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'LayerNorm_1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "LayerNorm_1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'MlpBlock_0', 'Dense_0', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MlpBlock_0", "key_type": 2}, {"key": "Dense_0", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'MlpBlock_0', 'Dense_0', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MlpBlock_0", "key_type": 2}, {"key": "Dense_0", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'MlpBlock_0', 'Dense_1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MlpBlock_0", "key_type": 2}, {"key": "Dense_1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'MlpBlock_0', 'Dense_1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MlpBlock_0", "key_type": 2}, {"key": "Dense_1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'key', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "key", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'key', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "key", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'out', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "out", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'out', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "out", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'query', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "query", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'query', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "query", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'value', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "value", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'value', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "value", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'img', 'embedding', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "embedding", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'img', 'embedding', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "embedding", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'img', 'head', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "head", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'img', 'head', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "head", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'img', 'pos_embedding')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "pos_embedding", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'llm', 'embedder', 'input_embedding')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "embedder", "key_type": 2}, {"key": "input_embedding", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'llm', 'final_norm', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "final_norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'llm', 'layers', 'attn', 'attn_vec_einsum', 'w')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "attn_vec_einsum", "key_type": 2}, {"key": "w", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'llm', 'layers', 'attn', 'kv_einsum', 'w')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "kv_einsum", "key_type": 2}, {"key": "w", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'llm', 'layers', 'attn', 'q_einsum', 'w')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "q_einsum", "key_type": 2}, {"key": "w", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'llm', 'layers', 'mlp', 'gating_einsum')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "gating_einsum", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'llm', 'layers', 'mlp', 'linear')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "linear", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'llm', 'layers', 'pre_attention_norm', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "pre_attention_norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'llm', 'layers', 'pre_ffw_norm', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "pre_ffw_norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'modality_embedder', 'Embed_0', 'embedding')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "modality_embedder", "key_type": 2}, {"key": "Embed_0", "key_type": 2}, {"key": "embedding", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'Dense_0', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "Dense_0", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'Dense_0', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "Dense_0", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit1', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit1', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit1', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit1', 'conv_proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit1', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit1', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit1', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit1', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit1', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit1', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit1', 'gn_proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit1', 'gn_proj', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit2', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit2', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit2', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit2', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit2', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit2', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit2', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit2', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block1', 'unit2', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit1', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit1', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit1', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit1', 'conv_proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit1', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit1', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit1', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit1', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit1', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit1', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit1', 'gn_proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit1', 'gn_proj', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit2', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit2', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit2', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit2', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit2', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit2', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit2', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit2', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block2', 'unit2', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit1', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit1', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit1', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit1', 'conv_proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit1', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit1', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit1', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit1', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit1', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit1', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit1', 'gn_proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit1', 'gn_proj', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit2', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit2', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit2', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit2', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit2', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit2', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit2', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit2', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block3', 'unit2', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit1', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit1', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit1', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit1', 'conv_proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit1', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit1', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit1', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit1', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit1', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit1', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit1', 'gn_proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit1', 'gn_proj', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit2', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit2', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit2', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit2', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit2', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit2', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit2', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit2', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'block4', 'unit2', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'conv_root', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "conv_root", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'gn_root', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "gn_root", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'spectro', 'gn_root', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "gn_root", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'start_image_digit_left')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "start_image_digit_left", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'start_image_digit_right')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "start_image_digit_right", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'start_image_primary')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "start_image_primary", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'start_image_wrist')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "start_image_wrist", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'start_mel_spectro')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "start_mel_spectro", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'start_modality_idx')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "start_modality_idx", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'mu', 'start_text')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "mu", "key_type": 2}, {"key": "start_text", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block0', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block0', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block0', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block0', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block0', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block0', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block0', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block0', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block0', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block0', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block0', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block0', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block1', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block1', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block1', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block1', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block1', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block1', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block1', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block1', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block1', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block1', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block1', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block1', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block10', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block10', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block10', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block10', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block10', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block10', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block10', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block10', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block10', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block10', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block10', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block10', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block11', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block11', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block11', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block11', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block11', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block11', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block11', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block11', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block11', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block11', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block11', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block11', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block2', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block2', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block2', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block2', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block2', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block2', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block2', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block2', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block2', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block2', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block2', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block2', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block3', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block3', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block3', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block3', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block3', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block3', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block3', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block3', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block3', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block3', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block3', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block3', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block4', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block4', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block4', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block4', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block4', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block4', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block4', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block4', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block4', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block4', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block4', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block4', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block5', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block5', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block5', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block5', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block5', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block5', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block5', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block5', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block5', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block5', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block5', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block5', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block6', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block6', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block6', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block6', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block6', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block6', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block6', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block6', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block6', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block6', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block6', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block6', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block7', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block7', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block7', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block7', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block7', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block7', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block7', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block7', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block7', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block7', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block7', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block7', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block8', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block8', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block8', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block8', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block8', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block8', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block8', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block8', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block8', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block8', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block8', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block8', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block9', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block9', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block9', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block9', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block9', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block9', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block9', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block9', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block9', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block9', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block9', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'block9', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'cls_token')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "cls_token", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'fc_norm', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "fc_norm", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'fc_norm', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "fc_norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'head', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "head", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'head', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "head", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'patch_embed', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "patch_embed", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'patch_embed', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "patch_embed", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'pool_func', 'attention_head', 'key', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "key", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'pool_func', 'attention_head', 'key', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "key", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'pool_func', 'attention_head', 'out', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "out", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'pool_func', 'attention_head', 'out', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "out", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'pool_func', 'attention_head', 'query', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "query", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'pool_func', 'attention_head', 'query', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "query", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'pool_func', 'attention_head', 'value', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "value", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'pool_func', 'attention_head', 'value', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "value", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'pool_func', 'readouts')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "readouts", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_left', 'pos_embed')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_left", "key_type": 2}, {"key": "pos_embed", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block0', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block0', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block0', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block0', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block0', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block0', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block0', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block0', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block0', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block0', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block0', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block0', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block0", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block1', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block1', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block1', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block1', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block1', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block1', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block1', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block1', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block1', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block1', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block1', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block1', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block10', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block10', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block10', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block10', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block10', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block10', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block10', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block10', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block10', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block10', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block10', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block10', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block10", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block11', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block11', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block11', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block11', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block11', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block11', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block11', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block11', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block11', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block11', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block11', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block11', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block11", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block2', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block2', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block2', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block2', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block2', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block2', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block2', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block2', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block2', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block2', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block2', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block2', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block3', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block3', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block3', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block3', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block3', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block3', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block3', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block3', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block3', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block3', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block3', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block3', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block4', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block4', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block4', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block4', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block4', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block4', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block4', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block4', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block4', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block4', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block4', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block4', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block5', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block5', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block5', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block5', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block5', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block5', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block5', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block5', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block5', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block5', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block5', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block5', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block5", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block6', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block6', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block6', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block6', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block6', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block6', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block6', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block6', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block6', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block6', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block6', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block6', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block6", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block7', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block7', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block7', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block7', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block7', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block7', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block7', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block7', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block7', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block7', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block7', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block7', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block7", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block8', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block8', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block8', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block8', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block8', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block8', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block8', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block8', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block8', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block8', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block8', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block8', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block8", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block9', 'attn', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block9', 'attn', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block9', 'attn', 'qkv', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block9', 'attn', 'qkv', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "qkv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block9', 'mlp', 'fc1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block9', 'mlp', 'fc1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block9', 'mlp', 'fc2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block9', 'mlp', 'fc2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block9', 'norm1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block9', 'norm1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block9', 'norm2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'block9', 'norm2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "block9", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'cls_token')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "cls_token", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'fc_norm', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "fc_norm", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'fc_norm', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "fc_norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'head', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "head", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'head', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "head", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'patch_embed', 'proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "patch_embed", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'patch_embed', 'proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "patch_embed", "key_type": 2}, {"key": "proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'pool_func', 'attention_head', 'key', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "key", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'pool_func', 'attention_head', 'key', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "key", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'pool_func', 'attention_head', 'out', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "out", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'pool_func', 'attention_head', 'out', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "out", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'pool_func', 'attention_head', 'query', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "query", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'pool_func', 'attention_head', 'query', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "query", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'pool_func', 'attention_head', 'value', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "value", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'pool_func', 'attention_head', 'value', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "attention_head", "key_type": 2}, {"key": "value", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'pool_func', 'readouts')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pool_func", "key_type": 2}, {"key": "readouts", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'digit_right', 'pos_embed')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "digit_right", "key_type": 2}, {"key": "pos_embed", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoder_norm', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoder_norm", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoder_norm', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoder_norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'LayerNorm_0', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "LayerNorm_0", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'LayerNorm_0', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "LayerNorm_0", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'LayerNorm_1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "LayerNorm_1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'LayerNorm_1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "LayerNorm_1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'MlpBlock_0', 'Dense_0', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MlpBlock_0", "key_type": 2}, {"key": "Dense_0", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'MlpBlock_0', 'Dense_0', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MlpBlock_0", "key_type": 2}, {"key": "Dense_0", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'MlpBlock_0', 'Dense_1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MlpBlock_0", "key_type": 2}, {"key": "Dense_1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'MlpBlock_0', 'Dense_1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MlpBlock_0", "key_type": 2}, {"key": "Dense_1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'key', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "key", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'key', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "key", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'out', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "out", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'out', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "out", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'query', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "query", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'query', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "query", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'value', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "value", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'img', 'Transformer', 'encoderblock', 'MultiHeadDotProductAttention_0', 'value', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "Transformer", "key_type": 2}, {"key": "encoderblock", "key_type": 2}, {"key": "MultiHeadDotProductAttention_0", "key_type": 2}, {"key": "value", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'img', 'embedding', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "embedding", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'img', 'embedding', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "embedding", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'img', 'head', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "head", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'img', 'head', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "head", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'img', 'pos_embedding')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "img", "key_type": 2}, {"key": "pos_embedding", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'llm', 'embedder', 'input_embedding')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "embedder", "key_type": 2}, {"key": "input_embedding", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'llm', 'final_norm', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "final_norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'llm', 'layers', 'attn', 'attn_vec_einsum', 'w')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "attn_vec_einsum", "key_type": 2}, {"key": "w", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'llm', 'layers', 'attn', 'kv_einsum', 'w')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "kv_einsum", "key_type": 2}, {"key": "w", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'llm', 'layers', 'attn', 'q_einsum', 'w')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "attn", "key_type": 2}, {"key": "q_einsum", "key_type": 2}, {"key": "w", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'llm', 'layers', 'mlp', 'gating_einsum')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "gating_einsum", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'llm', 'layers', 'mlp', 'linear')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "linear", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'llm', 'layers', 'pre_attention_norm', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "pre_attention_norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'llm', 'layers', 'pre_ffw_norm', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "pre_ffw_norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'modality_embedder', 'Embed_0', 'embedding')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "modality_embedder", "key_type": 2}, {"key": "Embed_0", "key_type": 2}, {"key": "embedding", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'Dense_0', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "Dense_0", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'Dense_0', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "Dense_0", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit1', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit1', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit1', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit1', 'conv_proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit1', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit1', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit1', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit1', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit1', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit1', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit1', 'gn_proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit1', 'gn_proj', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit2', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit2', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit2', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit2', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit2', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit2', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit2', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit2', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block1', 'unit2', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block1", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit1', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit1', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit1', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit1', 'conv_proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit1', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit1', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit1', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit1', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit1', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit1', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit1', 'gn_proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit1', 'gn_proj', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit2', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit2', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit2', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit2', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit2', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit2', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit2', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit2', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block2', 'unit2', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block2", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit1', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit1', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit1', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit1', 'conv_proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit1', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit1', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit1', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit1', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit1', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit1', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit1', 'gn_proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit1', 'gn_proj', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit2', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit2', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit2', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit2', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit2', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit2', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit2', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit2', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block3', 'unit2', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block3", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit1', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit1', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit1', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit1', 'conv_proj', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "conv_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit1', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit1', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit1', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit1', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit1', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit1', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit1', 'gn_proj', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit1', 'gn_proj', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit1", "key_type": 2}, {"key": "gn_proj", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit2', 'conv1', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv1", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit2', 'conv2', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv2", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit2', 'conv3', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "conv3", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit2', 'gn1', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit2', 'gn1', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn1", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit2', 'gn2', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit2', 'gn2', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn2", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit2', 'gn3', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'block4', 'unit2', 'gn3', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "block4", "key_type": 2}, {"key": "unit2", "key_type": 2}, {"key": "gn3", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'conv_root', 'kernel')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "conv_root", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'gn_root', 'bias')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "gn_root", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'spectro', 'gn_root', 'scale')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "spectro", "key_type": 2}, {"key": "gn_root", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'start_image_digit_left')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "start_image_digit_left", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'start_image_digit_right')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "start_image_digit_right", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'start_image_primary')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "start_image_primary", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'start_image_wrist')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "start_image_wrist", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'start_mel_spectro')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "start_mel_spectro", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'start_modality_idx')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "start_modality_idx", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '0', 'nu', 'start_text')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "0", "key_type": 1}, {"key": "nu", "key_type": 2}, {"key": "start_text", "key_type": 2}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '1')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "1", "key_type": 1}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}, "('opt_state', 'inner_state', 'inner_states', 'llm', 'inner_state', '1', '2')": {"key_metadata": [{"key": "opt_state", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "inner_states", "key_type": 2}, {"key": "llm", "key_type": 2}, {"key": "inner_state", "key_type": 2}, {"key": "1", "key_type": 1}, {"key": "2", "key_type": 1}], "value_metadata": {"value_type": "None", "skip_deserialize": true}}}, "use_zarr3": false}
\ No newline at end of file
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/_sharding b/FuSe_PaliVLA_model/15000/model_opt_state/_sharding
new file mode 100644
index 0000000000000000000000000000000000000000..9afaa7a4b25ade73ad2ba7df9d274f6d9cf9e444
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/_sharding
@@ -0,0 +1 @@
+{"b3B0X3N0YXRlLmNvdW50":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmh5cGVycGFyYW1zLmVtYmVkX2xlYXJuaW5nX3JhdGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmh5cGVycGFyYW1zLmltZ19sZWFybmluZ19yYXRl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmh5cGVycGFyYW1zLmxsbV9sZWFybmluZ19yYXRl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmh5cGVycGFyYW1zX3N0YXRlcy5lbWJlZF9sZWFybmluZ19yYXRlLmNvdW50":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmh5cGVycGFyYW1zX3N0YXRlcy5pbWdfbGVhcm5pbmdfcmF0ZS5jb3VudA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmh5cGVycGFyYW1zX3N0YXRlcy5sbG1fbGVhcm5pbmdfcmF0ZS5jb3VudA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAuY291bnQ=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazAuYXR0bi5wcm9qLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazAuYXR0bi5wcm9qLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazAuYXR0bi5xa3YuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazAuYXR0bi5xa3Yua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazAubWxwLmZjMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazAubWxwLmZjMS5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazAubWxwLmZjMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazAubWxwLmZjMi5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazAubm9ybTEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazAubm9ybTEuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazAubm9ybTIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazAubm9ybTIuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazEuYXR0bi5wcm9qLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazEuYXR0bi5wcm9qLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazEuYXR0bi5xa3YuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazEuYXR0bi5xa3Yua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazEubWxwLmZjMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazEubWxwLmZjMS5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazEubWxwLmZjMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazEubWxwLmZjMi5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazEubm9ybTEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazEubm9ybTEuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazEubm9ybTIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazEubm9ybTIuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazEwLm1scC5mYzEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazEwLm1scC5mYzEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazEwLm1scC5mYzIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazEwLm1scC5mYzIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazEwLm5vcm0xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazEwLm5vcm0xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazEwLm5vcm0yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazEwLm5vcm0yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazEwLmF0dG4ucHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazEwLmF0dG4ucHJvai5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazEwLmF0dG4ucWt2LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazEwLmF0dG4ucWt2Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazExLm1scC5mYzEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazExLm1scC5mYzEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazExLm1scC5mYzIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazExLm1scC5mYzIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazExLm5vcm0xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazExLm5vcm0xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazExLm5vcm0yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazExLm5vcm0yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazExLmF0dG4ucHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazExLmF0dG4ucHJvai5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazExLmF0dG4ucWt2LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazExLmF0dG4ucWt2Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazIuYXR0bi5wcm9qLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazIuYXR0bi5wcm9qLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazIuYXR0bi5xa3YuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazIuYXR0bi5xa3Yua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazIubWxwLmZjMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazIubWxwLmZjMS5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazIubWxwLmZjMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazIubWxwLmZjMi5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazIubm9ybTEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazIubm9ybTEuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazIubm9ybTIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazIubm9ybTIuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazMuYXR0bi5wcm9qLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazMuYXR0bi5wcm9qLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazMuYXR0bi5xa3YuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazMuYXR0bi5xa3Yua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazMubWxwLmZjMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazMubWxwLmZjMS5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazMubWxwLmZjMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazMubWxwLmZjMi5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazMubm9ybTEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazMubm9ybTEuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazMubm9ybTIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazMubm9ybTIuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazQuYXR0bi5wcm9qLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazQuYXR0bi5wcm9qLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazQuYXR0bi5xa3YuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazQuYXR0bi5xa3Yua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazQubWxwLmZjMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazQubWxwLmZjMS5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazQubWxwLmZjMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazQubWxwLmZjMi5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazQubm9ybTEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazQubm9ybTEuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazQubm9ybTIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazQubm9ybTIuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazUuYXR0bi5wcm9qLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazUuYXR0bi5wcm9qLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazUuYXR0bi5xa3YuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazUuYXR0bi5xa3Yua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazUubWxwLmZjMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazUubWxwLmZjMS5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazUubWxwLmZjMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazUubWxwLmZjMi5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazUubm9ybTEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazUubm9ybTEuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazUubm9ybTIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazUubm9ybTIuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazYuYXR0bi5wcm9qLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazYuYXR0bi5wcm9qLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazYuYXR0bi5xa3YuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazYuYXR0bi5xa3Yua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazYubWxwLmZjMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazYubWxwLmZjMS5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazYubWxwLmZjMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazYubWxwLmZjMi5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazYubm9ybTEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazYubm9ybTEuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazYubm9ybTIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazYubm9ybTIuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazcuYXR0bi5wcm9qLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazcuYXR0bi5wcm9qLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazcuYXR0bi5xa3YuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazcuYXR0bi5xa3Yua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazcubWxwLmZjMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazcubWxwLmZjMS5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazcubWxwLmZjMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazcubWxwLmZjMi5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazcubm9ybTEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazcubm9ybTEuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazcubm9ybTIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazcubm9ybTIuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazguYXR0bi5wcm9qLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazguYXR0bi5wcm9qLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazguYXR0bi5xa3YuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazguYXR0bi5xa3Yua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazgubWxwLmZjMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazgubWxwLmZjMS5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazgubWxwLmZjMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazgubWxwLmZjMi5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazgubm9ybTEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazgubm9ybTEuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazgubm9ybTIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazgubm9ybTIuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazkuYXR0bi5wcm9qLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazkuYXR0bi5wcm9qLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazkuYXR0bi5xa3YuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazkuYXR0bi5xa3Yua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazkubWxwLmZjMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazkubWxwLmZjMS5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazkubWxwLmZjMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazkubWxwLmZjMi5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazkubm9ybTEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazkubm9ybTEuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazkubm9ybTIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5ibG9jazkubm9ybTIuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5jbHNfdG9rZW4=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5mY19ub3JtLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5mY19ub3JtLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5oZWFkLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5oZWFkLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5wYXRjaF9lbWJlZC5wcm9qLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5wYXRjaF9lbWJlZC5wcm9qLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5wb29sX2Z1bmMuYXR0ZW50aW9uX2hlYWQua2V5LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5wb29sX2Z1bmMuYXR0ZW50aW9uX2hlYWQua2V5Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5wb29sX2Z1bmMuYXR0ZW50aW9uX2hlYWQub3V0LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5wb29sX2Z1bmMuYXR0ZW50aW9uX2hlYWQub3V0Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5wb29sX2Z1bmMuYXR0ZW50aW9uX2hlYWQucXVlcnkuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5wb29sX2Z1bmMuYXR0ZW50aW9uX2hlYWQucXVlcnkua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5wb29sX2Z1bmMuYXR0ZW50aW9uX2hlYWQudmFsdWUuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5wb29sX2Z1bmMuYXR0ZW50aW9uX2hlYWQudmFsdWUua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5wb29sX2Z1bmMucmVhZG91dHM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfbGVmdC5wb3NfZW1iZWQ=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuY2xzX3Rva2Vu":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s0Lm1scC5mYzEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s0Lm1scC5mYzEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s0Lm1scC5mYzIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s0Lm1scC5mYzIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s0Lm5vcm0xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s0Lm5vcm0xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s0Lm5vcm0yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s0Lm5vcm0yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s0LmF0dG4ucHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s0LmF0dG4ucHJvai5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s0LmF0dG4ucWt2LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s0LmF0dG4ucWt2Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s1Lm1scC5mYzEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s1Lm1scC5mYzEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s1Lm1scC5mYzIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s1Lm1scC5mYzIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s1Lm5vcm0xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s1Lm5vcm0xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s1Lm5vcm0yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s1Lm5vcm0yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s1LmF0dG4ucHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s1LmF0dG4ucHJvai5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s1LmF0dG4ucWt2LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s1LmF0dG4ucWt2Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s2Lm1scC5mYzEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s2Lm1scC5mYzEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s2Lm1scC5mYzIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s2Lm1scC5mYzIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s2Lm5vcm0xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s2Lm5vcm0xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s2Lm5vcm0yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s2Lm5vcm0yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s2LmF0dG4ucHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s2LmF0dG4ucHJvai5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s2LmF0dG4ucWt2LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s2LmF0dG4ucWt2Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s3Lm1scC5mYzEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s3Lm1scC5mYzEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s3Lm1scC5mYzIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s3Lm1scC5mYzIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s3Lm5vcm0xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s3Lm5vcm0xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s3Lm5vcm0yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s3Lm5vcm0yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s3LmF0dG4ucHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s3LmF0dG4ucHJvai5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s3LmF0dG4ucWt2LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s3LmF0dG4ucWt2Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s4Lm1scC5mYzEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s4Lm1scC5mYzEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s4Lm1scC5mYzIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s4Lm1scC5mYzIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s4Lm5vcm0xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s4Lm5vcm0xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s4Lm5vcm0yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s4Lm5vcm0yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s4LmF0dG4ucHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s4LmF0dG4ucHJvai5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s4LmF0dG4ucWt2LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s4LmF0dG4ucWt2Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s5Lm1scC5mYzEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s5Lm1scC5mYzEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s5Lm1scC5mYzIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s5Lm1scC5mYzIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s5Lm5vcm0xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s5Lm5vcm0xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s5Lm5vcm0yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s5Lm5vcm0yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s5LmF0dG4ucHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s5LmF0dG4ucHJvai5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s5LmF0dG4ucWt2LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2s5LmF0dG4ucWt2Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2swLm1scC5mYzEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2swLm1scC5mYzEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2swLm1scC5mYzIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2swLm1scC5mYzIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2swLm5vcm0xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2swLm5vcm0xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2swLm5vcm0yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2swLm5vcm0yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2swLmF0dG4ucHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2swLmF0dG4ucHJvai5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2swLmF0dG4ucWt2LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2swLmF0dG4ucWt2Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2sxLm1scC5mYzEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2sxLm1scC5mYzEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2sxLm1scC5mYzIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2sxLm1scC5mYzIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2sxLm5vcm0xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2sxLm5vcm0xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2sxLm5vcm0yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2sxLm5vcm0yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2sxLmF0dG4ucHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2sxLmF0dG4ucHJvai5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2sxLmF0dG4ucWt2LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2sxLmF0dG4ucWt2Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2sxMC5hdHRuLnByb2ouYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2sxMC5hdHRuLnByb2oua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2sxMC5hdHRuLnFrdi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2sxMC5hdHRuLnFrdi5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2sxMC5tbHAuZmMxLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2sxMC5tbHAuZmMxLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2sxMC5tbHAuZmMyLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2sxMC5tbHAuZmMyLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2sxMC5ub3JtMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2sxMC5ub3JtMS5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2sxMC5ub3JtMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2sxMC5ub3JtMi5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2sxMS5hdHRuLnByb2ouYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2sxMS5hdHRuLnByb2oua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2sxMS5hdHRuLnFrdi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2sxMS5hdHRuLnFrdi5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2sxMS5tbHAuZmMxLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2sxMS5tbHAuZmMxLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2sxMS5tbHAuZmMyLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2sxMS5tbHAuZmMyLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2sxMS5ub3JtMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2sxMS5ub3JtMS5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2sxMS5ub3JtMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2sxMS5ub3JtMi5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2syLm1scC5mYzEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2syLm1scC5mYzEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2syLm1scC5mYzIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2syLm1scC5mYzIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2syLm5vcm0xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2syLm5vcm0xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2syLm5vcm0yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2syLm5vcm0yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2syLmF0dG4ucHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2syLmF0dG4ucHJvai5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2syLmF0dG4ucWt2LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2syLmF0dG4ucWt2Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2szLm1scC5mYzEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2szLm1scC5mYzEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2szLm1scC5mYzIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2szLm1scC5mYzIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2szLm5vcm0xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2szLm5vcm0xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2szLm5vcm0yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2szLm5vcm0yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2szLmF0dG4ucHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2szLmF0dG4ucHJvai5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2szLmF0dG4ucWt2LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuYmxvY2szLmF0dG4ucWt2Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuZmNfbm9ybS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuZmNfbm9ybS5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuaGVhZC5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQuaGVhZC5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQucG9vbF9mdW5jLmF0dGVudGlvbl9oZWFkLm91dC5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQucG9vbF9mdW5jLmF0dGVudGlvbl9oZWFkLm91dC5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQucG9vbF9mdW5jLmF0dGVudGlvbl9oZWFkLmtleS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQucG9vbF9mdW5jLmF0dGVudGlvbl9oZWFkLmtleS5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQucG9vbF9mdW5jLmF0dGVudGlvbl9oZWFkLnF1ZXJ5LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQucG9vbF9mdW5jLmF0dGVudGlvbl9oZWFkLnF1ZXJ5Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQucG9vbF9mdW5jLmF0dGVudGlvbl9oZWFkLnZhbHVlLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQucG9vbF9mdW5jLmF0dGVudGlvbl9oZWFkLnZhbHVlLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQucG9vbF9mdW5jLnJlYWRvdXRz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQucG9zX2VtYmVk":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQucGF0Y2hfZW1iZWQucHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuZGlnaXRfcmlnaHQucGF0Y2hfZW1iZWQucHJvai5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUubW9kYWxpdHlfZW1iZWRkZXIuRW1iZWRfMC5lbWJlZGRpbmc=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5EZW5zZV8wLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5EZW5zZV8wLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [\"fsdp\", null]}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazEudW5pdDEuY29udjEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazEudW5pdDEuY29udjIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazEudW5pdDEuY29udjMua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazEudW5pdDEuY29udl9wcm9qLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazEudW5pdDEuZ24xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazEudW5pdDEuZ24xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazEudW5pdDEuZ24yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazEudW5pdDEuZ24yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazEudW5pdDEuZ24zLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazEudW5pdDEuZ24zLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazEudW5pdDEuZ25fcHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazEudW5pdDEuZ25fcHJvai5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazEudW5pdDIuY29udjEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazEudW5pdDIuY29udjIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazEudW5pdDIuY29udjMua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazEudW5pdDIuZ24xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazEudW5pdDIuZ24xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazEudW5pdDIuZ24yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazEudW5pdDIuZ24yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazEudW5pdDIuZ24zLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazEudW5pdDIuZ24zLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazIudW5pdDEuY29udjEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazIudW5pdDEuY29udjIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazIudW5pdDEuY29udjMua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazIudW5pdDEuY29udl9wcm9qLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazIudW5pdDEuZ24xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazIudW5pdDEuZ24xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazIudW5pdDEuZ24yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazIudW5pdDEuZ24yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazIudW5pdDEuZ24zLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazIudW5pdDEuZ24zLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazIudW5pdDEuZ25fcHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazIudW5pdDEuZ25fcHJvai5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazIudW5pdDIuY29udjEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazIudW5pdDIuY29udjIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazIudW5pdDIuY29udjMua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazIudW5pdDIuZ24xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazIudW5pdDIuZ24xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazIudW5pdDIuZ24yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazIudW5pdDIuZ24yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazIudW5pdDIuZ24zLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazIudW5pdDIuZ24zLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazMudW5pdDEuY29udjEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazMudW5pdDEuY29udjIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazMudW5pdDEuY29udjMua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazMudW5pdDEuY29udl9wcm9qLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazMudW5pdDEuZ24xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazMudW5pdDEuZ24xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazMudW5pdDEuZ24yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazMudW5pdDEuZ24yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazMudW5pdDEuZ24zLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazMudW5pdDEuZ24zLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazMudW5pdDEuZ25fcHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazMudW5pdDEuZ25fcHJvai5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazMudW5pdDIuY29udjEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazMudW5pdDIuY29udjIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazMudW5pdDIuY29udjMua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazMudW5pdDIuZ24xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazMudW5pdDIuZ24xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazMudW5pdDIuZ24yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazMudW5pdDIuZ24yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazMudW5pdDIuZ24zLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazMudW5pdDIuZ24zLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazQudW5pdDEuY29udjEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazQudW5pdDEuY29udjIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, null, \"fsdp\", null]}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazQudW5pdDEuY29udjMua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, null, null, \"fsdp\"]}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazQudW5pdDEuY29udl9wcm9qLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, null, null, \"fsdp\"]}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazQudW5pdDEuZ24xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazQudW5pdDEuZ24xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazQudW5pdDEuZ24yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazQudW5pdDEuZ24yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazQudW5pdDEuZ24zLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazQudW5pdDEuZ24zLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazQudW5pdDEuZ25fcHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazQudW5pdDEuZ25fcHJvai5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazQudW5pdDIuY29udjEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, null, \"fsdp\", null]}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazQudW5pdDIuY29udjIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, null, \"fsdp\", null]}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazQudW5pdDIuY29udjMua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, null, null, \"fsdp\"]}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazQudW5pdDIuZ24xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazQudW5pdDIuZ24xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazQudW5pdDIuZ24yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazQudW5pdDIuZ24yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazQudW5pdDIuZ24zLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5ibG9jazQudW5pdDIuZ24zLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5jb252X3Jvb3Qua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5nbl9yb290LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3BlY3Ryby5nbl9yb290LnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3RhcnRfaW1hZ2VfZGlnaXRfbGVmdA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3RhcnRfaW1hZ2VfZGlnaXRfcmlnaHQ=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3RhcnRfaW1hZ2VfcHJpbWFyeQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3RhcnRfaW1hZ2Vfd3Jpc3Q=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3RhcnRfbW9kYWxpdHlfaWR4":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3RhcnRfbWVsX3NwZWN0cm8=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubXUuc3RhcnRfdGV4dA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazAuYXR0bi5wcm9qLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazAuYXR0bi5wcm9qLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazAuYXR0bi5xa3YuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazAuYXR0bi5xa3Yua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazAubWxwLmZjMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazAubWxwLmZjMS5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazAubWxwLmZjMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazAubWxwLmZjMi5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazAubm9ybTEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazAubm9ybTEuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazAubm9ybTIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazAubm9ybTIuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazEuYXR0bi5wcm9qLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazEuYXR0bi5wcm9qLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazEuYXR0bi5xa3YuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazEuYXR0bi5xa3Yua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazEubWxwLmZjMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazEubWxwLmZjMS5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazEubWxwLmZjMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazEubWxwLmZjMi5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazEubm9ybTEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazEubm9ybTEuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazEubm9ybTIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazEubm9ybTIuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazEwLm1scC5mYzEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazEwLm1scC5mYzEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazEwLm1scC5mYzIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazEwLm1scC5mYzIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazEwLm5vcm0xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazEwLm5vcm0xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazEwLm5vcm0yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazEwLm5vcm0yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazEwLmF0dG4ucHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazEwLmF0dG4ucHJvai5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazEwLmF0dG4ucWt2LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazEwLmF0dG4ucWt2Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazExLm1scC5mYzEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazExLm1scC5mYzEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazExLm1scC5mYzIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazExLm1scC5mYzIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazExLm5vcm0xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazExLm5vcm0xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazExLm5vcm0yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazExLm5vcm0yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazExLmF0dG4ucHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazExLmF0dG4ucHJvai5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazExLmF0dG4ucWt2LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazExLmF0dG4ucWt2Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazIuYXR0bi5wcm9qLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazIuYXR0bi5wcm9qLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazIuYXR0bi5xa3YuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazIuYXR0bi5xa3Yua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazIubWxwLmZjMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazIubWxwLmZjMS5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazIubWxwLmZjMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazIubWxwLmZjMi5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazIubm9ybTEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazIubm9ybTEuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazIubm9ybTIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazIubm9ybTIuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazMuYXR0bi5wcm9qLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazMuYXR0bi5wcm9qLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazMuYXR0bi5xa3YuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazMuYXR0bi5xa3Yua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazMubWxwLmZjMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazMubWxwLmZjMS5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazMubWxwLmZjMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazMubWxwLmZjMi5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazMubm9ybTEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazMubm9ybTEuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazMubm9ybTIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazMubm9ybTIuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazQuYXR0bi5wcm9qLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazQuYXR0bi5wcm9qLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazQuYXR0bi5xa3YuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazQuYXR0bi5xa3Yua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazQubWxwLmZjMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazQubWxwLmZjMS5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazQubWxwLmZjMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazQubWxwLmZjMi5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazQubm9ybTEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazQubm9ybTEuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazQubm9ybTIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazQubm9ybTIuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazUuYXR0bi5wcm9qLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazUuYXR0bi5wcm9qLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazUuYXR0bi5xa3YuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazUuYXR0bi5xa3Yua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazUubWxwLmZjMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazUubWxwLmZjMS5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazUubWxwLmZjMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazUubWxwLmZjMi5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazUubm9ybTEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazUubm9ybTEuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazUubm9ybTIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazUubm9ybTIuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazYuYXR0bi5wcm9qLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazYuYXR0bi5wcm9qLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazYuYXR0bi5xa3YuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazYuYXR0bi5xa3Yua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazYubWxwLmZjMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazYubWxwLmZjMS5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazYubWxwLmZjMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazYubWxwLmZjMi5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazYubm9ybTEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazYubm9ybTEuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazYubm9ybTIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazYubm9ybTIuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazcuYXR0bi5wcm9qLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazcuYXR0bi5wcm9qLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazcuYXR0bi5xa3YuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazcuYXR0bi5xa3Yua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazcubWxwLmZjMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazcubWxwLmZjMS5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazcubWxwLmZjMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazcubWxwLmZjMi5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazcubm9ybTEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazcubm9ybTEuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazcubm9ybTIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazcubm9ybTIuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazguYXR0bi5wcm9qLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazguYXR0bi5wcm9qLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazguYXR0bi5xa3YuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazguYXR0bi5xa3Yua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazgubWxwLmZjMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazgubWxwLmZjMS5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazgubWxwLmZjMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazgubWxwLmZjMi5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazgubm9ybTEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazgubm9ybTEuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazgubm9ybTIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazgubm9ybTIuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazkuYXR0bi5wcm9qLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazkuYXR0bi5wcm9qLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazkuYXR0bi5xa3YuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazkuYXR0bi5xa3Yua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazkubWxwLmZjMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazkubWxwLmZjMS5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazkubWxwLmZjMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazkubWxwLmZjMi5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazkubm9ybTEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazkubm9ybTEuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazkubm9ybTIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5ibG9jazkubm9ybTIuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5jbHNfdG9rZW4=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5mY19ub3JtLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5mY19ub3JtLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5oZWFkLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5oZWFkLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5wYXRjaF9lbWJlZC5wcm9qLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5wYXRjaF9lbWJlZC5wcm9qLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5wb29sX2Z1bmMuYXR0ZW50aW9uX2hlYWQua2V5LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5wb29sX2Z1bmMuYXR0ZW50aW9uX2hlYWQua2V5Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5wb29sX2Z1bmMuYXR0ZW50aW9uX2hlYWQub3V0LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5wb29sX2Z1bmMuYXR0ZW50aW9uX2hlYWQub3V0Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5wb29sX2Z1bmMuYXR0ZW50aW9uX2hlYWQucXVlcnkuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5wb29sX2Z1bmMuYXR0ZW50aW9uX2hlYWQucXVlcnkua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5wb29sX2Z1bmMuYXR0ZW50aW9uX2hlYWQudmFsdWUuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5wb29sX2Z1bmMuYXR0ZW50aW9uX2hlYWQudmFsdWUua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5wb29sX2Z1bmMucmVhZG91dHM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfbGVmdC5wb3NfZW1iZWQ=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuY2xzX3Rva2Vu":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s0Lm1scC5mYzEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s0Lm1scC5mYzEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s0Lm1scC5mYzIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s0Lm1scC5mYzIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s0Lm5vcm0xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s0Lm5vcm0xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s0Lm5vcm0yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s0Lm5vcm0yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s0LmF0dG4ucHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s0LmF0dG4ucHJvai5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s0LmF0dG4ucWt2LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s0LmF0dG4ucWt2Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s1Lm1scC5mYzEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s1Lm1scC5mYzEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s1Lm1scC5mYzIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s1Lm1scC5mYzIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s1Lm5vcm0xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s1Lm5vcm0xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s1Lm5vcm0yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s1Lm5vcm0yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s1LmF0dG4ucHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s1LmF0dG4ucHJvai5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s1LmF0dG4ucWt2LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s1LmF0dG4ucWt2Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s2Lm1scC5mYzEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s2Lm1scC5mYzEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s2Lm1scC5mYzIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s2Lm1scC5mYzIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s2Lm5vcm0xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s2Lm5vcm0xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s2Lm5vcm0yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s2Lm5vcm0yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s2LmF0dG4ucHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s2LmF0dG4ucHJvai5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s2LmF0dG4ucWt2LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s2LmF0dG4ucWt2Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s3Lm1scC5mYzEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s3Lm1scC5mYzEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s3Lm1scC5mYzIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s3Lm1scC5mYzIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s3Lm5vcm0xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s3Lm5vcm0xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s3Lm5vcm0yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s3Lm5vcm0yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s3LmF0dG4ucHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s3LmF0dG4ucHJvai5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s3LmF0dG4ucWt2LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s3LmF0dG4ucWt2Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s4Lm1scC5mYzEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s4Lm1scC5mYzEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s4Lm1scC5mYzIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s4Lm1scC5mYzIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s4Lm5vcm0xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s4Lm5vcm0xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s4Lm5vcm0yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s4Lm5vcm0yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s4LmF0dG4ucHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s4LmF0dG4ucHJvai5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s4LmF0dG4ucWt2LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s4LmF0dG4ucWt2Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s5Lm1scC5mYzEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s5Lm1scC5mYzEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s5Lm1scC5mYzIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s5Lm1scC5mYzIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s5Lm5vcm0xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s5Lm5vcm0xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s5Lm5vcm0yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s5Lm5vcm0yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s5LmF0dG4ucHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s5LmF0dG4ucHJvai5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s5LmF0dG4ucWt2LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2s5LmF0dG4ucWt2Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2swLm1scC5mYzEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2swLm1scC5mYzEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2swLm1scC5mYzIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2swLm1scC5mYzIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2swLm5vcm0xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2swLm5vcm0xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2swLm5vcm0yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2swLm5vcm0yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2swLmF0dG4ucHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2swLmF0dG4ucHJvai5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2swLmF0dG4ucWt2LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2swLmF0dG4ucWt2Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2sxLm1scC5mYzEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2sxLm1scC5mYzEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2sxLm1scC5mYzIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2sxLm1scC5mYzIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2sxLm5vcm0xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2sxLm5vcm0xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2sxLm5vcm0yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2sxLm5vcm0yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2sxLmF0dG4ucHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2sxLmF0dG4ucHJvai5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2sxLmF0dG4ucWt2LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2sxLmF0dG4ucWt2Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2sxMC5hdHRuLnByb2ouYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2sxMC5hdHRuLnByb2oua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2sxMC5hdHRuLnFrdi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2sxMC5hdHRuLnFrdi5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2sxMC5tbHAuZmMxLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2sxMC5tbHAuZmMxLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2sxMC5tbHAuZmMyLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2sxMC5tbHAuZmMyLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2sxMC5ub3JtMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2sxMC5ub3JtMS5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2sxMC5ub3JtMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2sxMC5ub3JtMi5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2sxMS5hdHRuLnByb2ouYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2sxMS5hdHRuLnByb2oua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2sxMS5hdHRuLnFrdi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2sxMS5hdHRuLnFrdi5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2sxMS5tbHAuZmMxLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2sxMS5tbHAuZmMxLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2sxMS5tbHAuZmMyLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2sxMS5tbHAuZmMyLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2sxMS5ub3JtMS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2sxMS5ub3JtMS5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2sxMS5ub3JtMi5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2sxMS5ub3JtMi5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2syLm1scC5mYzEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2syLm1scC5mYzEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2syLm1scC5mYzIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2syLm1scC5mYzIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2syLm5vcm0xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2syLm5vcm0xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2syLm5vcm0yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2syLm5vcm0yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2syLmF0dG4ucHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2syLmF0dG4ucHJvai5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2syLmF0dG4ucWt2LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2syLmF0dG4ucWt2Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2szLm1scC5mYzEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2szLm1scC5mYzEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2szLm1scC5mYzIuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2szLm1scC5mYzIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2szLm5vcm0xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2szLm5vcm0xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2szLm5vcm0yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2szLm5vcm0yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2szLmF0dG4ucHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2szLmF0dG4ucHJvai5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2szLmF0dG4ucWt2LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuYmxvY2szLmF0dG4ucWt2Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuZmNfbm9ybS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuZmNfbm9ybS5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuaGVhZC5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQuaGVhZC5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQucG9vbF9mdW5jLmF0dGVudGlvbl9oZWFkLm91dC5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQucG9vbF9mdW5jLmF0dGVudGlvbl9oZWFkLm91dC5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQucG9vbF9mdW5jLmF0dGVudGlvbl9oZWFkLmtleS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQucG9vbF9mdW5jLmF0dGVudGlvbl9oZWFkLmtleS5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQucG9vbF9mdW5jLmF0dGVudGlvbl9oZWFkLnF1ZXJ5LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQucG9vbF9mdW5jLmF0dGVudGlvbl9oZWFkLnF1ZXJ5Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQucG9vbF9mdW5jLmF0dGVudGlvbl9oZWFkLnZhbHVlLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQucG9vbF9mdW5jLmF0dGVudGlvbl9oZWFkLnZhbHVlLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQucG9vbF9mdW5jLnJlYWRvdXRz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQucG9zX2VtYmVk":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQucGF0Y2hfZW1iZWQucHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuZGlnaXRfcmlnaHQucGF0Y2hfZW1iZWQucHJvai5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUubW9kYWxpdHlfZW1iZWRkZXIuRW1iZWRfMC5lbWJlZGRpbmc=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5EZW5zZV8wLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5EZW5zZV8wLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [\"fsdp\", null]}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazEudW5pdDEuY29udjEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazEudW5pdDEuY29udjIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazEudW5pdDEuY29udjMua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazEudW5pdDEuY29udl9wcm9qLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazEudW5pdDEuZ24xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazEudW5pdDEuZ24xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazEudW5pdDEuZ24yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazEudW5pdDEuZ24yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazEudW5pdDEuZ24zLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazEudW5pdDEuZ24zLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazEudW5pdDEuZ25fcHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazEudW5pdDEuZ25fcHJvai5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazEudW5pdDIuY29udjEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazEudW5pdDIuY29udjIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazEudW5pdDIuY29udjMua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazEudW5pdDIuZ24xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazEudW5pdDIuZ24xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazEudW5pdDIuZ24yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazEudW5pdDIuZ24yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazEudW5pdDIuZ24zLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazEudW5pdDIuZ24zLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazIudW5pdDEuY29udjEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazIudW5pdDEuY29udjIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazIudW5pdDEuY29udjMua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazIudW5pdDEuY29udl9wcm9qLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazIudW5pdDEuZ24xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazIudW5pdDEuZ24xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazIudW5pdDEuZ24yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazIudW5pdDEuZ24yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazIudW5pdDEuZ24zLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazIudW5pdDEuZ24zLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazIudW5pdDEuZ25fcHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazIudW5pdDEuZ25fcHJvai5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazIudW5pdDIuY29udjEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazIudW5pdDIuY29udjIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazIudW5pdDIuY29udjMua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazIudW5pdDIuZ24xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazIudW5pdDIuZ24xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazIudW5pdDIuZ24yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazIudW5pdDIuZ24yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazIudW5pdDIuZ24zLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazIudW5pdDIuZ24zLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazMudW5pdDEuY29udjEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazMudW5pdDEuY29udjIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazMudW5pdDEuY29udjMua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazMudW5pdDEuY29udl9wcm9qLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazMudW5pdDEuZ24xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazMudW5pdDEuZ24xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazMudW5pdDEuZ24yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazMudW5pdDEuZ24yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazMudW5pdDEuZ24zLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazMudW5pdDEuZ24zLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazMudW5pdDEuZ25fcHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazMudW5pdDEuZ25fcHJvai5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazMudW5pdDIuY29udjEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazMudW5pdDIuY29udjIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazMudW5pdDIuY29udjMua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazMudW5pdDIuZ24xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazMudW5pdDIuZ24xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazMudW5pdDIuZ24yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazMudW5pdDIuZ24yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazMudW5pdDIuZ24zLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazMudW5pdDIuZ24zLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazQudW5pdDEuY29udjEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazQudW5pdDEuY29udjIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, null, \"fsdp\", null]}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazQudW5pdDEuY29udjMua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, null, null, \"fsdp\"]}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazQudW5pdDEuY29udl9wcm9qLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, null, null, \"fsdp\"]}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazQudW5pdDEuZ24xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazQudW5pdDEuZ24xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazQudW5pdDEuZ24yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazQudW5pdDEuZ24yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazQudW5pdDEuZ24zLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazQudW5pdDEuZ24zLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazQudW5pdDEuZ25fcHJvai5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazQudW5pdDEuZ25fcHJvai5zY2FsZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazQudW5pdDIuY29udjEua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, null, \"fsdp\", null]}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazQudW5pdDIuY29udjIua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, null, \"fsdp\", null]}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazQudW5pdDIuY29udjMua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, null, null, \"fsdp\"]}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazQudW5pdDIuZ24xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazQudW5pdDIuZ24xLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazQudW5pdDIuZ24yLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazQudW5pdDIuZ24yLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazQudW5pdDIuZ24zLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5ibG9jazQudW5pdDIuZ24zLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5jb252X3Jvb3Qua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5nbl9yb290LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3BlY3Ryby5nbl9yb290LnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3RhcnRfaW1hZ2VfZGlnaXRfbGVmdA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3RhcnRfaW1hZ2VfZGlnaXRfcmlnaHQ=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3RhcnRfaW1hZ2VfcHJpbWFyeQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3RhcnRfaW1hZ2Vfd3Jpc3Q=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3RhcnRfbW9kYWxpdHlfaWR4":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3RhcnRfbWVsX3NwZWN0cm8=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5lbWJlZC5pbm5lcl9zdGF0ZS4xLjAubnUuc3RhcnRfdGV4dA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm11LmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyX25vcm0uYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm11LmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyX25vcm0uc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm11LmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTGF5ZXJOb3JtXzAuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm11LmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTGF5ZXJOb3JtXzAuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm11LmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTGF5ZXJOb3JtXzEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm11LmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTGF5ZXJOb3JtXzEuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm11LmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTWxwQmxvY2tfMC5EZW5zZV8wLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm11LmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTWxwQmxvY2tfMC5EZW5zZV8wLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, \"fsdp\", null]}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm11LmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTWxwQmxvY2tfMC5EZW5zZV8xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm11LmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTWxwQmxvY2tfMC5EZW5zZV8xLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, null, \"fsdp\"]}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm11LmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTXVsdGlIZWFkRG90UHJvZHVjdEF0dGVudGlvbl8wLm91dC5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm11LmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTXVsdGlIZWFkRG90UHJvZHVjdEF0dGVudGlvbl8wLm91dC5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, null, null, \"fsdp\"]}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm11LmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTXVsdGlIZWFkRG90UHJvZHVjdEF0dGVudGlvbl8wLmtleS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm11LmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTXVsdGlIZWFkRG90UHJvZHVjdEF0dGVudGlvbl8wLmtleS5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, \"fsdp\", null, null]}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm11LmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTXVsdGlIZWFkRG90UHJvZHVjdEF0dGVudGlvbl8wLnF1ZXJ5LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm11LmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTXVsdGlIZWFkRG90UHJvZHVjdEF0dGVudGlvbl8wLnF1ZXJ5Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, \"fsdp\", null, null]}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm11LmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTXVsdGlIZWFkRG90UHJvZHVjdEF0dGVudGlvbl8wLnZhbHVlLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm11LmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTXVsdGlIZWFkRG90UHJvZHVjdEF0dGVudGlvbl8wLnZhbHVlLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, \"fsdp\", null, null]}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm11LmltZy5lbWJlZGRpbmcuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm11LmltZy5lbWJlZGRpbmcua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm11LmltZy5oZWFkLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm11LmltZy5oZWFkLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, \"fsdp\"]}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm11LmltZy5wb3NfZW1iZWRkaW5n":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm51LmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyX25vcm0uYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm51LmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyX25vcm0uc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm51LmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTGF5ZXJOb3JtXzAuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm51LmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTGF5ZXJOb3JtXzAuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm51LmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTGF5ZXJOb3JtXzEuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm51LmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTGF5ZXJOb3JtXzEuc2NhbGU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm51LmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTWxwQmxvY2tfMC5EZW5zZV8wLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm51LmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTWxwQmxvY2tfMC5EZW5zZV8wLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, \"fsdp\", null]}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm51LmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTWxwQmxvY2tfMC5EZW5zZV8xLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm51LmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTWxwQmxvY2tfMC5EZW5zZV8xLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, null, \"fsdp\"]}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm51LmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTXVsdGlIZWFkRG90UHJvZHVjdEF0dGVudGlvbl8wLm91dC5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm51LmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTXVsdGlIZWFkRG90UHJvZHVjdEF0dGVudGlvbl8wLm91dC5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, null, null, \"fsdp\"]}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm51LmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTXVsdGlIZWFkRG90UHJvZHVjdEF0dGVudGlvbl8wLmtleS5iaWFz":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm51LmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTXVsdGlIZWFkRG90UHJvZHVjdEF0dGVudGlvbl8wLmtleS5rZXJuZWw=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, \"fsdp\", null, null]}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm51LmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTXVsdGlIZWFkRG90UHJvZHVjdEF0dGVudGlvbl8wLnF1ZXJ5LmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm51LmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTXVsdGlIZWFkRG90UHJvZHVjdEF0dGVudGlvbl8wLnF1ZXJ5Lmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, \"fsdp\", null, null]}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm51LmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTXVsdGlIZWFkRG90UHJvZHVjdEF0dGVudGlvbl8wLnZhbHVlLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm51LmltZy5UcmFuc2Zvcm1lci5lbmNvZGVyYmxvY2suTXVsdGlIZWFkRG90UHJvZHVjdEF0dGVudGlvbl8wLnZhbHVlLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, \"fsdp\", null, null]}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm51LmltZy5lbWJlZGRpbmcuYmlhcw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm51LmltZy5lbWJlZGRpbmcua2VybmVs":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm51LmltZy5oZWFkLmJpYXM=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm51LmltZy5oZWFkLmtlcm5lbA==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, \"fsdp\"]}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLm51LmltZy5wb3NfZW1iZWRkaW5n":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5pbWcuaW5uZXJfc3RhdGUuMS4wLmNvdW50":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5sbG0uaW5uZXJfc3RhdGUuMS4wLm11LmxsbS5lbWJlZGRlci5pbnB1dF9lbWJlZGRpbmc=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [\"fsdp\", null]}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5sbG0uaW5uZXJfc3RhdGUuMS4wLm11LmxsbS5maW5hbF9ub3JtLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5sbG0uaW5uZXJfc3RhdGUuMS4wLm11LmxsbS5sYXllcnMuYXR0bi5hdHRuX3ZlY19laW5zdW0udw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, null, null, \"fsdp\"]}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5sbG0uaW5uZXJfc3RhdGUuMS4wLm11LmxsbS5sYXllcnMuYXR0bi5rdl9laW5zdW0udw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, null, null, \"fsdp\", null]}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5sbG0uaW5uZXJfc3RhdGUuMS4wLm11LmxsbS5sYXllcnMuYXR0bi5xX2VpbnN1bS53":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, null, \"fsdp\", null]}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5sbG0uaW5uZXJfc3RhdGUuMS4wLm11LmxsbS5sYXllcnMubWxwLmdhdGluZ19laW5zdW0=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, null, null, \"fsdp\"]}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5sbG0uaW5uZXJfc3RhdGUuMS4wLm11LmxsbS5sYXllcnMubWxwLmxpbmVhcg==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, \"fsdp\", null]}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5sbG0uaW5uZXJfc3RhdGUuMS4wLm11LmxsbS5sYXllcnMucHJlX2F0dGVudGlvbl9ub3JtLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5sbG0uaW5uZXJfc3RhdGUuMS4wLm11LmxsbS5sYXllcnMucHJlX2Zmd19ub3JtLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5sbG0uaW5uZXJfc3RhdGUuMS4wLm51LmxsbS5lbWJlZGRlci5pbnB1dF9lbWJlZGRpbmc=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [\"fsdp\", null]}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5sbG0uaW5uZXJfc3RhdGUuMS4wLm51LmxsbS5maW5hbF9ub3JtLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5sbG0uaW5uZXJfc3RhdGUuMS4wLm51LmxsbS5sYXllcnMuYXR0bi5hdHRuX3ZlY19laW5zdW0udw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, null, null, \"fsdp\"]}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5sbG0uaW5uZXJfc3RhdGUuMS4wLm51LmxsbS5sYXllcnMuYXR0bi5rdl9laW5zdW0udw==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, null, null, \"fsdp\", null]}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5sbG0uaW5uZXJfc3RhdGUuMS4wLm51LmxsbS5sYXllcnMuYXR0bi5xX2VpbnN1bS53":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, null, \"fsdp\", null]}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5sbG0uaW5uZXJfc3RhdGUuMS4wLm51LmxsbS5sYXllcnMubWxwLmdhdGluZ19laW5zdW0=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, null, null, \"fsdp\"]}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5sbG0uaW5uZXJfc3RhdGUuMS4wLm51LmxsbS5sYXllcnMubWxwLmxpbmVhcg==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": [null, \"fsdp\", null]}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5sbG0uaW5uZXJfc3RhdGUuMS4wLm51LmxsbS5sYXllcnMucHJlX2F0dGVudGlvbl9ub3JtLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5sbG0uaW5uZXJfc3RhdGUuMS4wLm51LmxsbS5sYXllcnMucHJlX2Zmd19ub3JtLnNjYWxl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}","b3B0X3N0YXRlLmlubmVyX3N0YXRlLmlubmVyX3N0YXRlcy5sbG0uaW5uZXJfc3RhdGUuMS4wLmNvdW50":"{\"sharding_type\": \"NamedSharding\", \"shape\": [128], \"axis_names\": [\"fsdp\"], \"partition_spec\": []}"}
\ No newline at end of file
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/commit_success.txt b/FuSe_PaliVLA_model/15000/model_opt_state/commit_success.txt
new file mode 100644
index 0000000000000000000000000000000000000000..c1b9c11439b475e85e5692c6deb9c51e2bd54fa2
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/commit_success.txt
@@ -0,0 +1 @@
+Checkpoint commit was successful to gs://kyle-checkpoints-eu4/fuse-vla-checkpoints/golden-leaf-2/15000/model_opt_state
\ No newline at end of file
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/d/6589c3b32b3d56f12312443416d4fc29 b/FuSe_PaliVLA_model/15000/model_opt_state/d/6589c3b32b3d56f12312443416d4fc29
new file mode 100644
index 0000000000000000000000000000000000000000..6927a88d1c8f7e906a812628de86245ce3a5b303
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_opt_state/d/6589c3b32b3d56f12312443416d4fc29 differ
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_opt_state/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..5b93360ce0a3dc4a2893f552457d8ffe47b1daf9
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_opt_state/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_0/d/3963680224cb941a54cca49ba090aa01 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_0/d/3963680224cb941a54cca49ba090aa01
new file mode 100644
index 0000000000000000000000000000000000000000..15075425af9ae7681255513fb2d78f1813db67d5
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_0/d/3963680224cb941a54cca49ba090aa01 differ
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_0/d/3fad0236aa4a061db2e6b6c733f4a227 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_0/d/3fad0236aa4a061db2e6b6c733f4a227
new file mode 100644
index 0000000000000000000000000000000000000000..55ed1f6627b57e5e8968b2b7534b4fe0b59750b4
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_0/d/3fad0236aa4a061db2e6b6c733f4a227 differ
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_0/d/79e8496d6c7ce5fe6c133b0d16b51096 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_0/d/79e8496d6c7ce5fe6c133b0d16b51096
new file mode 100644
index 0000000000000000000000000000000000000000..72855fac70ec68dcf9ba103c216b9e9d9501ad5b
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_0/d/79e8496d6c7ce5fe6c133b0d16b51096 differ
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_0/d/923ae94e7f9287902d6e9876fccf2711 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_0/d/923ae94e7f9287902d6e9876fccf2711
new file mode 100644
index 0000000000000000000000000000000000000000..7a7a6d655dfcdd7c310b7c48ed45e97fc3edb575
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_0/d/923ae94e7f9287902d6e9876fccf2711 differ
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_0/d/acba0d9fb4b56b4829a3f8b4e6dde87c b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_0/d/acba0d9fb4b56b4829a3f8b4e6dde87c
new file mode 100644
index 0000000000000000000000000000000000000000..d337daf68e11e2574e17af1bb2da40fdf7e39eb1
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_0/d/acba0d9fb4b56b4829a3f8b4e6dde87c differ
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_0/d/b2b2d4192f8d1fb2721b4379ac131bb0 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_0/d/b2b2d4192f8d1fb2721b4379ac131bb0
new file mode 100644
index 0000000000000000000000000000000000000000..70b7ccb1fbf75d39c60d5a841610db18ed306b06
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_0/d/b2b2d4192f8d1fb2721b4379ac131bb0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90c55486664c0a4e08717db2723dc5a657e5e235802f979a8cc533bf4c560cff
+size 977958635
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_0/d/de842017dc94800c82512dc4d2c1a1cc b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_0/d/de842017dc94800c82512dc4d2c1a1cc
new file mode 100644
index 0000000000000000000000000000000000000000..b5e47c3a6e05e83dd5916fc7dd329dea388ef38d
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_0/d/de842017dc94800c82512dc4d2c1a1cc
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b39139fa6b6fa6167474e52e58aef657ec51e5b1f509a806356246fdda046bb
+size 74611856
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_0/d/f63e3ab5e40783a60d12f948c485ea29 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_0/d/f63e3ab5e40783a60d12f948c485ea29
new file mode 100644
index 0000000000000000000000000000000000000000..e52f1d019adc57309944aaffc875ec6a7c578c9b
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_0/d/f63e3ab5e40783a60d12f948c485ea29 differ
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_0/d/fe3a45e9f553db4dfb87f5b9b1264f7e b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_0/d/fe3a45e9f553db4dfb87f5b9b1264f7e
new file mode 100644
index 0000000000000000000000000000000000000000..9e62eb43ad3adb4546615037e575bfa0a231b77c
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_0/d/fe3a45e9f553db4dfb87f5b9b1264f7e differ
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_0/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_0/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..df34079cb40b59b92f5e01d6fa76ea607919cdc3
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_0/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_1/d/1c5a38f90f10f7fcef4697c958d1df0f b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_1/d/1c5a38f90f10f7fcef4697c958d1df0f
new file mode 100644
index 0000000000000000000000000000000000000000..424cb9cd8ccd450287f675c676abd07b07c9f3b0
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_1/d/1c5a38f90f10f7fcef4697c958d1df0f
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42d338385cce65a1725acb36fd36c7da6150041ccfe1afb02aa32d259f7835eb
+size 552106520
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_1/d/df49db5b82884be4ae0db6fde1a70509 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_1/d/df49db5b82884be4ae0db6fde1a70509
new file mode 100644
index 0000000000000000000000000000000000000000..60d3ea3dac56aebc97431bf4db4c8945ceb04b48
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_1/d/df49db5b82884be4ae0db6fde1a70509
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c81d8b58cc288d29ed1b74ac3f7896f3316aea79a779a4ebc1192682f0ca1a7
+size 123664581
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_1/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_1/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..f3bd73712dfaa2350c7b4adcb498b66110dcdd10
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_1/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_10/d/169303b9063568ebc21d7c44c6b68487 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_10/d/169303b9063568ebc21d7c44c6b68487
new file mode 100644
index 0000000000000000000000000000000000000000..9f3f43e756716be2b7eadeed9b49a0543800a413
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_10/d/169303b9063568ebc21d7c44c6b68487
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:736740a9632ece64fb5c17eae8f959f20851b3795c28e6b52de04e8f720902cf
+size 559922711
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_10/d/2f8fa5c13c4935a6f5dcee803db11d27 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_10/d/2f8fa5c13c4935a6f5dcee803db11d27
new file mode 100644
index 0000000000000000000000000000000000000000..98b0d5512f016b832fa722c6386d30095ce5f524
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_10/d/2f8fa5c13c4935a6f5dcee803db11d27
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec41744afcd9851a8d1e62578b58f91c9d5d25069191c72cd86fca00e8c79ff6
+size 116075197
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_10/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_10/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..ca94dcad986d337b18bd9f8b5c0fc7c2a8d9de23
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_10/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_11/d/41fd39581f2fdbb7c7f88967ad30b88f b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_11/d/41fd39581f2fdbb7c7f88967ad30b88f
new file mode 100644
index 0000000000000000000000000000000000000000..cfef90776dcca60668abec3f538be37f3732756f
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_11/d/41fd39581f2fdbb7c7f88967ad30b88f
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b118c9ed60e811d11c03e728380907ea9e0a294ed9da86a23f9b074f1dfc272b
+size 115958674
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_11/d/e77cbf5e2beeb8eda67bc2dd4c9bd050 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_11/d/e77cbf5e2beeb8eda67bc2dd4c9bd050
new file mode 100644
index 0000000000000000000000000000000000000000..bd1e1286f877349f2342784112ceaffa5c1dac2c
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_11/d/e77cbf5e2beeb8eda67bc2dd4c9bd050
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7746ecab26358103d77eeb66e455464bdb390d88934b8dc7826ba5e61ce753a2
+size 560416009
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_11/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_11/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..d2365c77eaa48db34b93f094a93bc02b08a40bc6
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_11/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_12/d/a0c1df1ccbf3948ec20816930f5242a2 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_12/d/a0c1df1ccbf3948ec20816930f5242a2
new file mode 100644
index 0000000000000000000000000000000000000000..ded65b125262f6b34926bfec510aa6442ccc7abd
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_12/d/a0c1df1ccbf3948ec20816930f5242a2
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cdf044e2908dabd21016ea9ab0854879301a3781110fb7520355e8896bc3be86
+size 554518354
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_12/d/f901b15c02b6030ae3aa85e2c14c9879 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_12/d/f901b15c02b6030ae3aa85e2c14c9879
new file mode 100644
index 0000000000000000000000000000000000000000..3c3937a3aac3f8ef3e9141e88ef2cb878a5df214
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_12/d/f901b15c02b6030ae3aa85e2c14c9879
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a67403dec03b13f7a641015cfc081d5e7b980e46d3f9478d0c63019decf30550
+size 121515714
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_12/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_12/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..f1dabd11299b2ee418ded6885e260a62e7d778ba
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_12/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_13/d/e5fa87781576945c2c9a6c1273f8b1b8 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_13/d/e5fa87781576945c2c9a6c1273f8b1b8
new file mode 100644
index 0000000000000000000000000000000000000000..cb2d80d38c78311cce31cccb85ac8f2b24517cd1
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_13/d/e5fa87781576945c2c9a6c1273f8b1b8
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4cd08caed2578c074cea04e81e32f5a134966d6b06d8fc2a2ba5895a42730ac
+size 562120539
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_13/d/f60ec11c15c5df1107d7996760e221a5 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_13/d/f60ec11c15c5df1107d7996760e221a5
new file mode 100644
index 0000000000000000000000000000000000000000..447f220edf7f3ab4948908eb7199d5f69f293646
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_13/d/f60ec11c15c5df1107d7996760e221a5
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8bbcbc2df32f2cac81e0bf0264a17a957ccfd96e54f1a0a36a1d9b75c5fc2cce
+size 113723219
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_13/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_13/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..64932227d9ae12f8511375aa16e175e887ec6293
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_13/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_14/d/2c344f840a241a8b8fadec56eed49204 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_14/d/2c344f840a241a8b8fadec56eed49204
new file mode 100644
index 0000000000000000000000000000000000000000..ac3d0a0c9a8b62e9f9292e169a9de4d5c731ab9a
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_14/d/2c344f840a241a8b8fadec56eed49204
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0992a24e150d0f3cca43629289a2d15e93446f0798a78ec972f0004f5e9db91
+size 122208578
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_14/d/3c62b1eef6e37ee7dcd23f28eaa55f27 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_14/d/3c62b1eef6e37ee7dcd23f28eaa55f27
new file mode 100644
index 0000000000000000000000000000000000000000..90b74c21b50630d61df050ac2c35fe876d848f05
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_14/d/3c62b1eef6e37ee7dcd23f28eaa55f27
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0c809460ed1ca7a1572dbbe98e78aaa6c91741cd7192509066d6b09c92cc904
+size 554011535
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_14/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_14/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..dfd2f71cf827a4893a1bbd001eacae6efd008def
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_14/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_15/d/4287aca8e4da1e46359ce16ad7537941 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_15/d/4287aca8e4da1e46359ce16ad7537941
new file mode 100644
index 0000000000000000000000000000000000000000..de870cc3b711260cc5e32677b45f53f2759e9c6d
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_15/d/4287aca8e4da1e46359ce16ad7537941
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e67d77d1c4b90fad2519fb2ae4d169b0315374f7208c843f316c0b47849f69f
+size 112310232
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_15/d/d9691fde608ced02a7180e4f75fb1e47 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_15/d/d9691fde608ced02a7180e4f75fb1e47
new file mode 100644
index 0000000000000000000000000000000000000000..b770846ce616beeb40e5dc9b6302dd7560ccc4b1
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_15/d/d9691fde608ced02a7180e4f75fb1e47
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4fa0790fbab8ccad5d8d0f8d8548286d91dc64578a09273848ac117e7637220
+size 564358038
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_15/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_15/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..9978fc602569a5e456656fc92a309d191279aa08
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_15/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_16/d/0fcea0b63e2a564a7b97a319d25a9b7f b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_16/d/0fcea0b63e2a564a7b97a319d25a9b7f
new file mode 100644
index 0000000000000000000000000000000000000000..05e4aaebcd0f20169c015af3923363ac92f2a4b3
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_16/d/0fcea0b63e2a564a7b97a319d25a9b7f
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7eb042d5685079dc662e2420b29c3e222dbbb0383aef9a93521a6944e907dca9
+size 117511377
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_16/d/4c7521528de273013c1e222445927a7a b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_16/d/4c7521528de273013c1e222445927a7a
new file mode 100644
index 0000000000000000000000000000000000000000..409fca2952c879a55cfb5edd2bcc0820528b7989
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_16/d/4c7521528de273013c1e222445927a7a
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71f39db65edb2302e340eeaaec73a57c02c01a18c9357963d2412c9f1c8be746
+size 558501370
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_16/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_16/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..0abf0037d1a7f4040d912c4b6f43b4357ef001ae
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_16/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_17/d/3dca904f7fa08e662b395cbf58a1c0b4 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_17/d/3dca904f7fa08e662b395cbf58a1c0b4
new file mode 100644
index 0000000000000000000000000000000000000000..5f41916d0ef237891aeb898a382f6d46519f25fd
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_17/d/3dca904f7fa08e662b395cbf58a1c0b4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fdd70d9fff3b1030c380e8ae0232cf3b73647363d91cf526c5c88085e75dba33
+size 552164065
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_17/d/6f0938986ed04df2c37c113af1b7a470 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_17/d/6f0938986ed04df2c37c113af1b7a470
new file mode 100644
index 0000000000000000000000000000000000000000..dc9268c06b420e79400b9920b4339e96448912de
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_17/d/6f0938986ed04df2c37c113af1b7a470
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f191e9ef5d139bba7cd0072415357c35a982ccfdf77216e8c929e9bbe8c6336
+size 123804980
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_17/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_17/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..7cdb37df77084452c568abc07a4dd6cecb77d43a
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_17/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_18/d/1499f8858662865a9ecc8dc296d1d189 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_18/d/1499f8858662865a9ecc8dc296d1d189
new file mode 100644
index 0000000000000000000000000000000000000000..f7bcbce23a9f17690f00f57ee8224f397c7a7766
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_18/d/1499f8858662865a9ecc8dc296d1d189
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31ac3913c46851ba9651ba0b0bcce70368ecd62b3c48a54bdfc7b9893d42af72
+size 122071433
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_18/d/cfddcc8fdccb06464e380b341619ea23 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_18/d/cfddcc8fdccb06464e380b341619ea23
new file mode 100644
index 0000000000000000000000000000000000000000..0e9dbbbbdf1e151edaffa31781627a0c5bee6a2e
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_18/d/cfddcc8fdccb06464e380b341619ea23
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fcbe8045b4683a9551b643d398de6adc8d1af9918ecbece2b1da0fc0f388b92a
+size 554042143
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_18/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_18/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..09d0eb63ca42da4e503364672a38f4d60dfd30f7
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_18/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_19/d/1e2b73aeb58359924fc9486156fea0e3 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_19/d/1e2b73aeb58359924fc9486156fea0e3
new file mode 100644
index 0000000000000000000000000000000000000000..d0fff0bfd294b055e88a59cec3be1733766251c4
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_19/d/1e2b73aeb58359924fc9486156fea0e3
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4680bebb78063494cedf8b4d1f0991fb913dbd90252322fb0920a205d0b70770
+size 558554263
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_19/d/f6287fa59d763f6bf5dd981251eb5caa b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_19/d/f6287fa59d763f6bf5dd981251eb5caa
new file mode 100644
index 0000000000000000000000000000000000000000..af7de28fbd69241c70bb4654c9e6fd992d796acb
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_19/d/f6287fa59d763f6bf5dd981251eb5caa
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e5b4440b549398cb17c9867fe2dc328d0cd19a06547579e5456e212bc83ada30
+size 118222609
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_19/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_19/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..093d5f134d6e8422c6ac51a68b1106d5ba4659b4
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_19/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_2/d/3083e52ee48a8d93e95a3603946ebf2e b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_2/d/3083e52ee48a8d93e95a3603946ebf2e
new file mode 100644
index 0000000000000000000000000000000000000000..368610ba1a0db6879e394b4f6c320f59828dbeef
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_2/d/3083e52ee48a8d93e95a3603946ebf2e
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:430697fc113b673833ec8289235bbce021237e819f575cf73c91e76d9c4064f8
+size 556078701
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_2/d/98be3aabfc8519b6a51b69134d263876 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_2/d/98be3aabfc8519b6a51b69134d263876
new file mode 100644
index 0000000000000000000000000000000000000000..bc73be95af0f37ab3d4ad5d5f4d7eb045a6064f7
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_2/d/98be3aabfc8519b6a51b69134d263876
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f385b17ac7c5f05cad12125edc620c7e34a9971a0577d3f7ddcddfaf00cce34
+size 120106223
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_2/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_2/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..20156085ba445d456518d861cf7c0cfc7e1363d9
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_2/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_20/d/3186551c62f14827729fbce166596418 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_20/d/3186551c62f14827729fbce166596418
new file mode 100644
index 0000000000000000000000000000000000000000..b0cc8ff278475de76d75d6df2f218432a69c5ed6
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_20/d/3186551c62f14827729fbce166596418
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f3472cde84ea6b2ebabb2b7fdbcc6b0637d1004c7474e8928ddaac5eeea7a03
+size 549917216
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_20/d/9aecae8ebf718fd8dce25c4cd24a2cdb b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_20/d/9aecae8ebf718fd8dce25c4cd24a2cdb
new file mode 100644
index 0000000000000000000000000000000000000000..2a982be88832811606db358139ab2231babb4347
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_20/d/9aecae8ebf718fd8dce25c4cd24a2cdb
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:711572f0874b7b0ff2497a5799fc817408008ce8da4a577b42ccf90f1034a3e1
+size 126023910
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_20/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_20/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..12c393878847a4fff96273590350210f44517414
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_20/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_21/d/0514c5a3422c9a87562309b692e67ca8 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_21/d/0514c5a3422c9a87562309b692e67ca8
new file mode 100644
index 0000000000000000000000000000000000000000..d6c54e88e91437ae561383abd89d08e1eac947f9
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_21/d/0514c5a3422c9a87562309b692e67ca8
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6bbce7306cc819c1dc708857a2f4b3a3251eba9a0729f5023911051f0be61221
+size 125974023
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_21/d/27752287f91c2645c49b80c26b73e4fc b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_21/d/27752287f91c2645c49b80c26b73e4fc
new file mode 100644
index 0000000000000000000000000000000000000000..92a57e5661b181d6ccf7142f8de7154eae2a705a
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_21/d/27752287f91c2645c49b80c26b73e4fc
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e82ed84fa590c0b759d93edcbc3c3f02018f6cb3e8798c50e169103678132e6
+size 549941124
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_21/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_21/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..952037aa36728eaaf95f6f9590edf281dd7cfaab
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_21/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_22/d/0b2c316b3fc5dd112bef35f7a9f91c93 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_22/d/0b2c316b3fc5dd112bef35f7a9f91c93
new file mode 100644
index 0000000000000000000000000000000000000000..78883852f2bc80935bbff226e4c128dec291e9f9
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_22/d/0b2c316b3fc5dd112bef35f7a9f91c93
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a314c9880fe9ed0c9ac5a939d8d112c803a95f38f50532edaa86a853ae54cee0
+size 550320387
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_22/d/5f6eac589f8368de171498b46e53ec98 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_22/d/5f6eac589f8368de171498b46e53ec98
new file mode 100644
index 0000000000000000000000000000000000000000..e5d9ec5a3f4e93fa66a0bca439383082c1ed4c32
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_22/d/5f6eac589f8368de171498b46e53ec98
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6d5cd8be0fd0c651837e8750504706016b56ab224be7b0c55c0a202b1788f4a
+size 125849795
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_22/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_22/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..6b00aac3939b89f5a2a8806eda854ecbfc439a39
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_22/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_23/d/a5b715ca6f77998a463f8ba8aebfbe76 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_23/d/a5b715ca6f77998a463f8ba8aebfbe76
new file mode 100644
index 0000000000000000000000000000000000000000..257f54bc96c05ec35c93120a31442bef064e9517
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_23/d/a5b715ca6f77998a463f8ba8aebfbe76
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4bdcf1447b7fccc28a009e5df2f5b553ec5355790990a77697fe362f67186235
+size 550350150
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_23/d/b42eafcfa226c95a8e567ce0ef95ac98 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_23/d/b42eafcfa226c95a8e567ce0ef95ac98
new file mode 100644
index 0000000000000000000000000000000000000000..a02459e2a036caff7a3edb39805b757335135853
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_23/d/b42eafcfa226c95a8e567ce0ef95ac98
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b3574e389836965949094b10f7ab1b231348ed91f77a0f8bb6aaa1cfe4b0c7be
+size 126130755
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_23/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_23/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..624653e19b4a2c542aed50781c3a4e672f391721
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_23/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_24/d/6c3ccc76533c19087492265b1a738415 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_24/d/6c3ccc76533c19087492265b1a738415
new file mode 100644
index 0000000000000000000000000000000000000000..28687bbf203b39c44952296871ea5939d05fe729
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_24/d/6c3ccc76533c19087492265b1a738415
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc52c4c2e699ea5685e7af99a0324562b1215e39746621b61a4593eadca84b78
+size 125839213
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_24/d/ba39e146acfe623519c669f38854e58f b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_24/d/ba39e146acfe623519c669f38854e58f
new file mode 100644
index 0000000000000000000000000000000000000000..3ea4a9ebba2b24b5f5dcfa6a0a33fa773ecf3fe5
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_24/d/ba39e146acfe623519c669f38854e58f
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95365fc1dc89f16992ad4d3f0d2d8bf96524fa6bca57f630e5cda72bbba7afd6
+size 549887795
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_24/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_24/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..0f4b350c06519084cfd9b8e48b66a30e5b65df70
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_24/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_25/d/1430470b859a65a9029b1e14a7c09dc4 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_25/d/1430470b859a65a9029b1e14a7c09dc4
new file mode 100644
index 0000000000000000000000000000000000000000..43a3773f7bd831d263a745cccf759a7adf31573e
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_25/d/1430470b859a65a9029b1e14a7c09dc4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cac9aebf14221ac816cf192931d2b58d553c2d5d8cfd3da71032e245df9601e4
+size 553925174
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_25/d/6a64d36f4a3d51c5e190716ac9e28c71 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_25/d/6a64d36f4a3d51c5e190716ac9e28c71
new file mode 100644
index 0000000000000000000000000000000000000000..3b0b45dd7bf25d6243d1e5d61f98cf27d93bba62
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_25/d/6a64d36f4a3d51c5e190716ac9e28c71
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1619ab7a64bebb7a1e5634ccb49205891f467ab37cfc11a8516cbf4fea950683
+size 121919634
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_25/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_25/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..6505137a2a69a99d994f9e8b054f7e7acdd3bc9e
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_25/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_26/d/10baf404782bca68d121552866e1eb70 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_26/d/10baf404782bca68d121552866e1eb70
new file mode 100644
index 0000000000000000000000000000000000000000..6edfc35528f083d39636a7a0f178674353d4eef8
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_26/d/10baf404782bca68d121552866e1eb70
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df0916686863afabee586e5473d8bc6afe19944298fa31723f1bd41a921794b4
+size 113872950
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_26/d/2af55ac925c869dc84dfa101faf79526 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_26/d/2af55ac925c869dc84dfa101faf79526
new file mode 100644
index 0000000000000000000000000000000000000000..7b95b80ec6476a9e569989bad22babd5053e8c24
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_26/d/2af55ac925c869dc84dfa101faf79526
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c9d58f787ec621ba4f7f0987d55009f11ff18416ee07cd3f8089103a85c6847
+size 562317302
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_26/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_26/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..7b50fe5c9feb7921b654a89b5b48afc4b3204486
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_26/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_27/d/2830f658bc973c353bbf25ab21535be6 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_27/d/2830f658bc973c353bbf25ab21535be6
new file mode 100644
index 0000000000000000000000000000000000000000..e6e3f4759f6545b92603d0b3905e7e1fa4fae9f2
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_27/d/2830f658bc973c353bbf25ab21535be6
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f9a8cd489c7593a60c9e44c1f5d6d3c7e95504daee5564eb8fbc907847fdefd
+size 166722354
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_27/d/dc136ca01b5b69effd00c82f7b67a732 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_27/d/dc136ca01b5b69effd00c82f7b67a732
new file mode 100644
index 0000000000000000000000000000000000000000..27f310ef50cdff00350c6a9dbf544317f831745b
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_27/d/dc136ca01b5b69effd00c82f7b67a732
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:30eb55f18a0def1e46f4d25a215b29cc7d6b7ce3f12e46aa6659fb4c0433ad05
+size 509435744
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_27/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_27/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..bcc4e64437d8fc3f6d3a853248a07fc9e2f8a454
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_27/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_28/d/51fb3b7591a83c0cae47674a1562662c b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_28/d/51fb3b7591a83c0cae47674a1562662c
new file mode 100644
index 0000000000000000000000000000000000000000..eb5de502bfb8e71befb3b5fbd94cbd3e35002653
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_28/d/51fb3b7591a83c0cae47674a1562662c
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d625eea56e98b8d29410817f792f7724f740f54f80723a034ff195352e98e2d
+size 560337973
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_28/d/cb5b6764d8410576a6bcffbb43aee7c3 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_28/d/cb5b6764d8410576a6bcffbb43aee7c3
new file mode 100644
index 0000000000000000000000000000000000000000..e41b69d2801cbd1d613d492bfb0341ab98bebedc
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_28/d/cb5b6764d8410576a6bcffbb43aee7c3
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:552354954bda49753b10d74d4dc7e4bbc1259dd8b730f2af9b0691199c4c8453
+size 115410061
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_28/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_28/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..e4573070a8158f02e70327ae5f7fe61de98b5e27
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_28/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_29/d/7b18440f014da88801f09c4051f683ff b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_29/d/7b18440f014da88801f09c4051f683ff
new file mode 100644
index 0000000000000000000000000000000000000000..f4525eb06a892aa3c0d575c1325a73ae588e9bd9
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_29/d/7b18440f014da88801f09c4051f683ff
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d7225ac6c1c3b113775774790291cd1ac5c787d3ff66beeb8e3d3cd1e8ebd58
+size 117625841
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_29/d/b4cf19f1739cee0917d904b114e0c4d0 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_29/d/b4cf19f1739cee0917d904b114e0c4d0
new file mode 100644
index 0000000000000000000000000000000000000000..80e7916a972f11f2ae362dbe12ceccabeedb69fa
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_29/d/b4cf19f1739cee0917d904b114e0c4d0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3587f7fb541000e6a3a27545084d9d4bc17c51ad1308d2f9c4027a5c7ce6ecd
+size 558309559
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_29/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_29/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..b62498a8f5162b23763982fbfb6ca84b7b774f82
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_29/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_3/d/6584ee49a934b4100bf64dd34b963896 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_3/d/6584ee49a934b4100bf64dd34b963896
new file mode 100644
index 0000000000000000000000000000000000000000..5b16c8472b2e02440c55ed3d8ff82a8ea8a7a8d0
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_3/d/6584ee49a934b4100bf64dd34b963896
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb029548d029b96863df5a10352f0e8a156486ade83933382e762162b2a31b68
+size 128031974
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_3/d/e0e8cb5ea3b67de7ad9d977f0e788dbb b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_3/d/e0e8cb5ea3b67de7ad9d977f0e788dbb
new file mode 100644
index 0000000000000000000000000000000000000000..9c64dd65e61c25d51ea9e76aa498f2dcd21d9c81
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_3/d/e0e8cb5ea3b67de7ad9d977f0e788dbb
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2cb7362a785cee1fe7d7812041779e7934f6ed76a5ea108019ec8a30bbf0594
+size 548190978
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_3/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_3/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..856ce9ef59456a860fb9d91e16d85ac7745b4973
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_3/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_30/d/7a53e2fb9d2ff318ecb3ecb2e9cdd3d0 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_30/d/7a53e2fb9d2ff318ecb3ecb2e9cdd3d0
new file mode 100644
index 0000000000000000000000000000000000000000..c6718a5a9a53bf5035929e6f7c26a9af2ba8978d
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_30/d/7a53e2fb9d2ff318ecb3ecb2e9cdd3d0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:80ecc4ee423dca046e03b4de8b14c277ffd3f3ec63f11c04c5a58d846cf6adf9
+size 124371020
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_30/d/c89415f376323bbbb2d6e57e49d819d7 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_30/d/c89415f376323bbbb2d6e57e49d819d7
new file mode 100644
index 0000000000000000000000000000000000000000..306764474c6d637c70adb6247e6e6816a29c2853
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_30/d/c89415f376323bbbb2d6e57e49d819d7
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:04d298fa7a25bd0fe07fd5496f0f25a51b23386b2bbca6026dd6f5c9f7fce649
+size 551984740
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_30/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_30/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..188484f1348ee4bbf92a1c5c85d735c40d90ff63
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_30/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_31/d/67e947717027c4201b83f1c312a3dfd2 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_31/d/67e947717027c4201b83f1c312a3dfd2
new file mode 100644
index 0000000000000000000000000000000000000000..e4ae65c0dce5b3bb03a2134d77cc6796042162ae
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_31/d/67e947717027c4201b83f1c312a3dfd2
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2c98e07c92235de94df25857b319707937bbc91730850c01ba9a932dff5d106
+size 554010928
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_31/d/c85296320cd7fdf6c375bdca6d3e2959 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_31/d/c85296320cd7fdf6c375bdca6d3e2959
new file mode 100644
index 0000000000000000000000000000000000000000..34276571e26e8a6a11480b1a707ac9337ac513be
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_31/d/c85296320cd7fdf6c375bdca6d3e2959
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3177baabe5360ed88f3519be72c883b3afa1217baec22572a69c0de74e892e93
+size 121489841
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_31/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_31/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..f53163ed81989fe9484d8cfbc6e9bff9b1839a6f
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_31/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_4/d/1bae1a66cc02f46a28f25aed3c72f408 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_4/d/1bae1a66cc02f46a28f25aed3c72f408
new file mode 100644
index 0000000000000000000000000000000000000000..dcd6853bfbf7e8264d5d2fff48b50256bc2403bc
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_4/d/1bae1a66cc02f46a28f25aed3c72f408
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:01989e015360bddc0385e0f4eb2b0960b198daef4c57732b553ee8c2111242a5
+size 556486519
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_4/d/2d40d34f5260ec4069b288b2a0c514a0 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_4/d/2d40d34f5260ec4069b288b2a0c514a0
new file mode 100644
index 0000000000000000000000000000000000000000..5dfa4c2811f93b9103aa6520a560685378857a10
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_4/d/2d40d34f5260ec4069b288b2a0c514a0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b81d66acf30efc0b87cf9058e8b8b4e983846d95c79b59ff8679796f070d263
+size 119880665
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_4/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_4/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..5dc895f0ae95ef80a3e69251eaff519d092266cc
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_4/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_5/d/df547fbbcb53d9b85fc7dfa1d7414930 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_5/d/df547fbbcb53d9b85fc7dfa1d7414930
new file mode 100644
index 0000000000000000000000000000000000000000..deed0878ff924ba69ce86396e0f1273e53fc5f5a
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_5/d/df547fbbcb53d9b85fc7dfa1d7414930
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76dbc912813b517adf5d10bb6d064833d86e01ce742b9bde0115e89559a25e86
+size 116045253
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_5/d/e7269d185aa10563d65edf4cad96d800 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_5/d/e7269d185aa10563d65edf4cad96d800
new file mode 100644
index 0000000000000000000000000000000000000000..bc95621042d6592884f963b28061016c6ffecf0c
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_5/d/e7269d185aa10563d65edf4cad96d800
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:272a1da6eacd056a5674f673568216bd59b339e84b127324f3b36062169edb81
+size 559923251
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_5/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_5/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..e48e306a934f02d07af8b917acbbfc38f6554a62
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_5/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_6/d/33c9c913825f7d2a61f0739d28514b32 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_6/d/33c9c913825f7d2a61f0739d28514b32
new file mode 100644
index 0000000000000000000000000000000000000000..37f8eb4f528e644c336670d5c7183ceff6b45b97
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_6/d/33c9c913825f7d2a61f0739d28514b32
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5437ecbcfccff03cb65befc361d983a6266f83feec11eaea0737fbfe86fbb83b
+size 555726810
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_6/d/63c9faf5a9d0c5c9998e3240a363ee0a b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_6/d/63c9faf5a9d0c5c9998e3240a363ee0a
new file mode 100644
index 0000000000000000000000000000000000000000..441d4e8040b8c411605634d7c996b81f95df7adc
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_6/d/63c9faf5a9d0c5c9998e3240a363ee0a
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29acc2f2607835d4583affa185c910d028d28eb9451dc33db261621110e2e8f3
+size 120334008
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_6/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_6/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..c46244d543226a02c658b60be0a497628ff0206e
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_6/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_7/d/32843871a03f7e535f0d732c2c7b4752 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_7/d/32843871a03f7e535f0d732c2c7b4752
new file mode 100644
index 0000000000000000000000000000000000000000..4df8a14c68fb3a81dc16eca7a8f0ea59228049a9
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_7/d/32843871a03f7e535f0d732c2c7b4752
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c71e2670202677ae27ccc00a04c3e3159825f771211423aa6e10a494af917ff3
+size 552715472
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_7/d/38bf128cb85362140971dc8eccff77e8 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_7/d/38bf128cb85362140971dc8eccff77e8
new file mode 100644
index 0000000000000000000000000000000000000000..0b67e893f8d73c8dacaf0fcf5b44374241a0fbd9
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_7/d/38bf128cb85362140971dc8eccff77e8
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d037599973f2eac9d393cda4c162be92c7a4b0369871d6f71b6c95b0febd962
+size 123674515
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_7/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_7/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..0a1009623042936bf3678177198b0e01c45897ab
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_7/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_8/d/45852443b4b7194b9e1f571c6bd8f1dd b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_8/d/45852443b4b7194b9e1f571c6bd8f1dd
new file mode 100644
index 0000000000000000000000000000000000000000..82b26c564bfcb14bd6d159e4e78b32a3f579e1e3
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_8/d/45852443b4b7194b9e1f571c6bd8f1dd
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42d0afd9655f00f797260c3ac632234612a67c58100ec4ecf1fa4769f36daec4
+size 556485012
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_8/d/858301ae6f38ba94adffab05b1387cba b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_8/d/858301ae6f38ba94adffab05b1387cba
new file mode 100644
index 0000000000000000000000000000000000000000..932e68bfe0000b1f17ff0558a7abdc5596572c9d
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_8/d/858301ae6f38ba94adffab05b1387cba
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ae0c9e5346233be2f5eb88f6ba3f34c48638afc0df2cce49c60ebd349bf7d15
+size 119779584
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_8/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_8/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..6cf196b20a675f65b55e2cbee906363ca7902bd7
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_8/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_9/d/04559f6daf0bf3aaa85938d897fd2336 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_9/d/04559f6daf0bf3aaa85938d897fd2336
new file mode 100644
index 0000000000000000000000000000000000000000..8f4203b7b0c0af48e0eb48659c585cf71309c48e
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_9/d/04559f6daf0bf3aaa85938d897fd2336
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e482f4ea108370b09a8e5cddb19399094409e9acb41ebec5e115b907ceab18d
+size 568096596
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_9/d/e6a824634176aed6f106a2d71b705cf4 b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_9/d/e6a824634176aed6f106a2d71b705cf4
new file mode 100644
index 0000000000000000000000000000000000000000..b73973d8f98255625d1450a657476ecf2cae948a
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_9/d/e6a824634176aed6f106a2d71b705cf4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:333d2ca61637e879f093ca239c8e6d3644b89d16835cffac17dd6eb5692b9513
+size 107768927
diff --git a/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_9/manifest.ocdbt b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_9/manifest.ocdbt
new file mode 100644
index 0000000000000000000000000000000000000000..adde2a0ca784b8b76bd2406c79c9dd410ce60e1c
Binary files /dev/null and b/FuSe_PaliVLA_model/15000/model_opt_state/ocdbt.process_9/manifest.ocdbt differ
diff --git a/FuSe_PaliVLA_model/15000/model_optimizer_spec/commit_success.txt b/FuSe_PaliVLA_model/15000/model_optimizer_spec/commit_success.txt
new file mode 100644
index 0000000000000000000000000000000000000000..cbdc045d1c5d7d05e7b7ca7fc2acc13687936cb7
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_optimizer_spec/commit_success.txt
@@ -0,0 +1 @@
+Checkpoint commit was successful to gs://kyle-checkpoints-eu4/fuse-vla-checkpoints/golden-leaf-2/15000/model_optimizer_spec
\ No newline at end of file
diff --git a/FuSe_PaliVLA_model/15000/model_optimizer_spec/metadata b/FuSe_PaliVLA_model/15000/model_optimizer_spec/metadata
new file mode 100644
index 0000000000000000000000000000000000000000..68aae93a77c1f8a807085032458a7307bc57d1d5
--- /dev/null
+++ b/FuSe_PaliVLA_model/15000/model_optimizer_spec/metadata
@@ -0,0 +1 @@
+{"__ctor": "palivla.load_model.make_optimizer", "config": {"embed_optimizer_kwargs": {"b1": 0.9, "b2": 0.99, "grad_norm_clip": 10.0, "init_learning_rate": 0, "learning_rate": 1e-05, "warmup_steps": 1000, "weight_decay": 0.0}, "img_optimizer_kwargs": {"b1": 0.9, "b2": 0.99, "grad_norm_clip": 10.0, "init_learning_rate": 0, "learning_rate": 1e-05, "warmup_steps": 2000, "weight_decay": 5e-06}, "llm_optimizer_kwargs": {"b1": 0.9, "b2": 0.99, "grad_norm_clip": 10.0, "init_learning_rate": 0, "learning_rate": 1e-05, "warmup_steps": 2000, "weight_decay": 5e-06}, "num_steps": 40000, "optimizer": "adamw"}}
\ No newline at end of file