new

README.md

@@ -1,3 +1,102 @@
----
-license: mit
----
+---
+license: apache-2.0
+language:
+- en
+- zh
+pipeline_tag: token-classification
+---
+# bert-chunker-0.2
+
+
+## Introduction
+
+bert-chunker-0.2 is a text chunker based on BERT with a classifier head to predict the start token of chunks (for use in RAG, etc), and using a sliding window it cuts documents of any size into chunks. It was finetuned on top of [nreimers/MiniLM-L6-H384-uncased](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2). 
+
+bert-chunker-0.2 is a new experimental version of bert-chunker. It's enhanced for article structures, by reaching a balance between semantics chunking and structure chunking. It is a 0.1:0.9 linear weight merging of a trained semantics chunker and a trained structure chunker.
+
+
+## Quickstart
+Run the following:
+
+```python
+import torch
+from transformers import AutoConfig,AutoTokenizer,BertForTokenClassification
+import math
+model_path="tim1900/bert-chunker-0.2"
+tokenizer = AutoTokenizer.from_pretrained(
+    model_path,
+    padding_side="right",
+    model_max_length=255,
+    trust_remote_code=True,
+)
+config = AutoConfig.from_pretrained(
+    model_path,
+    trust_remote_code=True,
+)
+device = 'cpu'
+model = BertForTokenClassification.from_pretrained(model_path, ).to(device)
+def chunk_text(model,text:str, tokenizer, prob_threshold=0.5)->list[str]:
+# slide context window
+    MAX_TOKENS=255
+    tokens=tokenizer(text, return_tensors="pt",truncation=False)
+    input_ids=tokens['input_ids']
+    attention_mask=tokens['attention_mask'][:,0:MAX_TOKENS]
+    attention_mask=attention_mask.to(model.device)
+    CLS=input_ids[:,0].unsqueeze(0)
+    SEP=input_ids[:,-1].unsqueeze(0)
+    input_ids=input_ids[:,1:-1]
+    model.eval()
+    split_str_poses=[]
+    
+    token_pos = []
+
+    windows_start =0
+    windows_end= 0
+    logits_threshold = math.log(1/prob_threshold-1)
+    
+    print(f'Processing {input_ids.shape[1]} tokens...')
+    while windows_end <= input_ids.shape[1]:
+        windows_end= windows_start + MAX_TOKENS-2
+
+        ids=torch.cat((CLS, input_ids[:,windows_start:windows_end],SEP),1)
+
+        ids=ids.to(model.device)
+        
+        output=model(input_ids=ids,attention_mask=torch.ones(1, ids.shape[1],device=model.device))
+        logits = output['logits'][:, 1:-1,:]
+        chunk_decision = (logits[:,:,1]>(logits[:,:,0]-logits_threshold))
+        greater_rows_indices = torch.where(chunk_decision)[1].tolist()
+
+        # null or not
+        if len(greater_rows_indices)>0 and (not (greater_rows_indices[0] == 0 and len(greater_rows_indices)==1)):
+
+            split_str_pos=[tokens.token_to_chars(sp + windows_start + 1).start for sp in greater_rows_indices]
+            token_pos +=[sp + windows_start + 1 for sp in greater_rows_indices]
+            split_str_poses += split_str_pos
+
+            windows_start = greater_rows_indices[-1] + windows_start
+
+        else:
+
+            windows_start = windows_end
+
+    substrings = [text[i:j] for i, j in zip([0] + split_str_poses, split_str_poses+[len(text)])]
+    token_pos = [0] + token_pos
+    return substrings,token_pos
+
+
+
+text='''In the heart of the bustling city, where towering skyscrapers touch the clouds and the symphony 
+    of honking cars never ceases, Sarah, an aspiring novelist, found solace in the quiet corners of the ancient library 
+    Surrounded by shelves that whispered stories of centuries past, she crafted her own world with words, oblivious to the rush outside Dr.Alexander Thompson, aboard the spaceship 'Pandora's Venture', was en route to the newly discovered exoplanet Zephyr-7. 
+    As the lead astrobiologist of the expedition, his mission was to uncover signs of microbial life within the planet's subterranean ice caves. 
+    With each passing light year, the anticipation of unraveling secrets that could alter humanity's
+     understanding of life in the universe grew ever stronger.'''
+
+chunks, token_pos=chunk_text(model,text, tokenizer, prob_threshold=0.5)
+
+# print chunks
+for i, (c,t) in enumerate(zip(chunks,token_pos)):
+    print(f'-----chunk: {i}----token_idx: {t}--------')
+    print(c)
+```

config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "/data/pretrained_model",
+  "architectures": [
+    "BertForTokenClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 384,
+  "initializer_range": 0.02,
+  "intermediate_size": 1536,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 6,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.46.3",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 30522
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a87cabf3305e1e1a41712a73829c4a4305fb6a7df0e3487aac02afb2e636994
+size 90276408

special_tokens_map.json

@@ -0,0 +1,37 @@
+{
+  "cls_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "[MASK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,64 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "101": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "102": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "103": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": false,
+  "cls_token": "[CLS]",
+  "do_basic_tokenize": true,
+  "do_lower_case": true,
+  "mask_token": "[MASK]",
+  "max_length": 128,
+  "model_max_length": 255,
+  "never_split": null,
+  "pad_to_multiple_of": null,
+  "pad_token": "[PAD]",
+  "pad_token_type_id": 0,
+  "padding_side": "right",
+  "sep_token": "[SEP]",
+  "stride": 0,
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
+  "unk_token": "[UNK]"
+}