UniversalDependenciesPipeline

README.md

@@ -84,3 +84,11 @@ print(nlp("Hai cái đầu thì tốt hơn một."))
 ```
 
 with [ufal.chu-liu-edmonds](https://pypi.org/project/ufal.chu-liu-edmonds/) and [ViNLP](https://pypi.org/project/ViNLP/).
+Or without them:
+
+```
+from transformers import pipeline
+nlp=pipeline("universal-dependencies","KoichiYasuoka/phobert-base-vietnamese-ud-goeswith",trust_remote_code=True,aggregation_strategy="simple")
+print(nlp("Hai cái đầu thì tốt hơn một."))
+```
+

config.json

@@ -5,6 +5,11 @@
   "attention_probs_dropout_prob": 0.1,
   "bos_token_id": 0,
   "classifier_dropout": null,
+  "custom_pipelines": {
+    "universal-dependencies": {
+      "impl": "ud.UniversalDependenciesPipeline"
+    }
+  },
   "eos_token_id": 2,
   "gradient_checkpointing": false,
   "hidden_act": "gelu",

ud.py

@@ -0,0 +1,85 @@
+from transformers import TokenClassificationPipeline
+
+class UniversalDependenciesPipeline(TokenClassificationPipeline):
+  def preprocess(self,sentence,offset_mapping=None):
+    from tokenizers.pre_tokenizers import Whitespace
+    t=[]
+    for k,(s,e) in Whitespace().pre_tokenize_str(sentence):
+      if t==[]:
+        t.append((k,(s,e)))
+      else:
+        j=t[-1][0]+"_"+k
+        if self.tokenizer.convert_tokens_to_ids(j)!=self.tokenizer.unk_token_id:
+          t[-1]=(j,(t[-1][1][0],e))
+        else:
+          t.append((k,(s,e)))
+    r=super().preprocess(sentence=" ".join(i for i,j in t))
+    m=[(0,0)]+[j for i,j in t]+[(0,0)]
+    w=self.tokenizer.convert_ids_to_tokens(r["input_ids"][0])
+    if len(m)!=len(w):
+      for i,j in enumerate(w):
+        if j.endswith("@@"):
+          s,e=m[i]
+          m.insert(i+1,(s+len(j)-2,e))
+          m[i]=(s,s+len(j)-2)
+    r["offset_mapping"]=m
+    r["sentence"]=sentence
+    return r
+  def _forward(self,model_inputs):
+    import torch
+    v=model_inputs["input_ids"][0].tolist()
+    with torch.no_grad():
+      e=self.model(input_ids=torch.tensor([v[0:i]+[self.tokenizer.mask_token_id]+v[i+1:]+[j] for i,j in enumerate(v[1:-1],1)]))
+    return {"logits":e.logits[:,1:-2,:],**model_inputs}
+  def postprocess(self,model_outputs,**kwargs):
+    import numpy
+    e=model_outputs["logits"].numpy()
+    r=[1 if i==0 else -1 if j.endswith("|root") else 0 for i,j in sorted(self.model.config.id2label.items())]
+    e+=numpy.where(numpy.add.outer(numpy.identity(e.shape[0]),r)==0,0,numpy.nan)
+    g=self.model.config.label2id["X|_|goeswith"]
+    r=numpy.tri(e.shape[0])
+    for i in range(e.shape[0]):
+      for j in range(i+2,e.shape[1]):
+        r[i,j]=r[i,j-1] if numpy.nanargmax(e[i,j-1])==g else 1
+    e[:,:,g]+=numpy.where(r==0,0,numpy.nan)
+    m,p=numpy.nanmax(e,axis=2),numpy.nanargmax(e,axis=2)
+    h=self.chu_liu_edmonds(m)
+    z=[i for i,j in enumerate(h) if i==j]
+    if len(z)>1:
+      k,h=z[numpy.nanargmax(m[z,z])],numpy.nanmin(m)-numpy.nanmax(m)
+      m[:,z]+=[[0 if j in z and (i!=j or i==k) else h for i in z] for j in range(m.shape[0])]
+      h=self.chu_liu_edmonds(m)
+    v=[(s,e) for s,e in model_outputs["offset_mapping"] if s<e]
+    q=[self.model.config.id2label[p[j,i]].split("|") for i,j in enumerate(h)]
+    g="aggregation_strategy" in kwargs and kwargs["aggregation_strategy"]!="none"
+    if g:
+      for i,j in reversed(list(enumerate(q[1:],1))):
+        if j[-1]=="goeswith" and set([t[-1] for t in q[h[i]+1:i+1]])=={"goeswith"}:
+          h=[b if i>b else b-1 for a,b in enumerate(h) if i!=a]
+          v[i-1]=(v[i-1][0],v.pop(i)[1])
+          q.pop(i)
+    t=model_outputs["sentence"].replace("\n"," ")
+    u="# text = "+t+"\n"
+    for i,(s,e) in enumerate(v):
+      u+="\t".join([str(i+1),t[s:e],t[s:e] if g else "_",q[i][0],"_","|".join(q[i][1:-1]),str(0 if h[i]==i else h[i]+1),q[i][-1],"_","_" if i+1<len(v) and e<v[i+1][0] else "SpaceAfter=No"])+"\n"
+    return u+"\n"
+  def chu_liu_edmonds(self,matrix):
+    import numpy
+    h=numpy.nanargmax(matrix,axis=0)
+    x=[-1 if i==j else j for i,j in enumerate(h)]
+    for b in [lambda x,i,j:-1 if i not in x else x[i],lambda x,i,j:-1 if j<0 else x[j]]:
+      y=[]
+      while x!=y:
+        y=list(x)
+        for i,j in enumerate(x):
+          x[i]=b(x,i,j)
+      if max(x)<0:
+        return h
+    y,x=[i for i,j in enumerate(x) if j==max(x)],[i for i,j in enumerate(x) if j<max(x)]
+    z=matrix-numpy.nanmax(matrix,axis=0)
+    m=numpy.block([[z[x,:][:,x],numpy.nanmax(z[x,:][:,y],axis=1).reshape(len(x),1)],[numpy.nanmax(z[y,:][:,x],axis=0),numpy.nanmax(z[y,y])]])
+    k=[j if i==len(x) else x[j] if j<len(x) else y[numpy.nanargmax(z[y,x[i]])] for i,j in enumerate(self.chu_liu_edmonds(m))]
+    h=[j if i in y else k[x.index(i)] for i,j in enumerate(h)]
+    i=y[numpy.nanargmax(z[x[k[-1]],y] if k[-1]<len(x) else z[y,y])]
+    h[i]=x[k[-1]] if k[-1]<len(x) else i
+    return h