messing around with differing settings.

config.json

@@ -16,13 +16,11 @@
     "AutoModelForSequenceClassification": "modeling_lsg_bert.LSGBertForSequenceClassification",
     "AutoModelForTokenClassification": "modeling_lsg_bert.LSGBertForTokenClassification"
   },
+  "base_model_prefix": "lsg",
   "block_size": 128,
+  "pool_with_global": true,
   "sparse_block_size": 128,
   "sparsity_factor": 2,
-  "base_model_prefix": "lsg",
-  "sparsity_type": "norm",
-  "is_decoder": false,
-  "pool_with_global": true,
   "num_global_tokens": 1,
   "classifier_dropout": null,
   "hidden_act": "gelu",
@@ -39,6 +37,7 @@
   "num_hidden_layers": 24,
   "pad_token_id": 0,
   "position_embedding_type": "absolute",
+  "sparsity_type": "norm",
   "torch_dtype": "float32",
   "transformers_version": "4.30.2",
   "type_vocab_size": 2,