messing around with differing settings.

modeling_lsg_bert.py

@@ -59,16 +59,16 @@ class LSGBertConfig(BertConfig):
 
     def __init__(
         self,
+        adaptive=False,
         base_model_prefix="lsg",
         block_size=0,
-        sparse_block_size=128,
-        adaptive=True,
-        sparsity_factor=2,
         lsh_num_pre_rounds=1,
+        sparse_block_size=0,
         mask_first_token=False,
-        num_global_tokens=1,
-        pool_with_global=True,
-        sparsity_type="none",
+        num_global_tokens=0,
+        pool_with_global=False,
+        sparsity_factor=1,
+        sparsity_type="non3",
         **kwargs
         ):
         """Constructs LSGBertConfig."""
@@ -98,20 +98,20 @@ class LSGBertConfig(BertConfig):
                 "[WARNING CONFIG]: sparsity_factor > encoder_attention_heads is not recommended for stride/block_stride sparsity"
             )
         
-        if self.num_global_tokens < 1:
-            logger.warning(
-                "[WARNING CONFIG]: num_global_tokens < 1 is not compatible, setting num_global_tokens=1"
-            )
-            self.num_global_tokens = 1
+        # if self.num_global_tokens < 1:
+        #     logger.warning(
+        #         "[WARNING CONFIG]: num_global_tokens < 1 is not compatible, setting num_global_tokens=1"
+        #     )
+        #     self.num_global_tokens = 1
         elif self.num_global_tokens > 512:
             logger.warning(
                 "[WARNING CONFIG]: num_global_tokens > 512 is not allowed, setting num_global_tokens=512"
             )
             self.num_global_tokens = 512
         
-        if self.sparsity_factor > 0:
-            assert self.block_size % self.sparsity_factor == 0, "[ERROR CONFIG]: block_size must be divisible by sparsity_factor"
-            assert self.block_size//self.sparsity_factor >= 1, "[ERROR CONFIG]: make sure block_size >= sparsity_factor"
+        # if self.sparsity_factor > 0:
+        #     assert self.block_size % self.sparsity_factor == 0, "[ERROR CONFIG]: block_size must be divisible by sparsity_factor"
+        #     assert self.block_size//self.sparsity_factor >= 1, "[ERROR CONFIG]: make sure block_size >= sparsity_factor"
             
         if self.mask_first_token and not pool_with_global:
             logger.warning(