multi-syllable words

maker.py

@@ -21,36 +21,22 @@ class UDgoeswithDataset(object):
         if len(t)==10 and t[0].isdecimal():
           c.append(t)
         elif c!=[]:
-          d=list(c)
-          v=tokenizer([t[1].replace(" ","_") for t in c],add_special_tokens=False)["input_ids"]
-          for i in range(len(v)-1,-1,-1):
-            for j in range(1,len(v[i])):
-              c.insert(i+1,[c[i][0],"_","_","X","_","_",c[i][0],"goeswith","_","_"])
-          y=["0"]+[t[0] for t in c]
-          h=[i if t[6]=="0" else y.index(t[6]) for i,t in enumerate(c,1)]
-          p,v=[t[3]+"|"+t[5]+"|"+t[7] for t in c],sum(v,[])
-          if len(v)<tokenizer.model_max_length-3:
-            self.ids.append([cls]+v+[sep])
-            self.tags.append([dep]+p+[dep])
-            label=set(sum([self.tags[-1],list(label)],[]))
-            for i,k in enumerate(v):
-              self.ids.append([cls]+v[0:i]+[msk]+v[i+1:]+[sep,k])
-              self.tags.append([dep]+[t if h[j]==i+1 else dep for j,t in enumerate(p)]+[dep,dep])
-          c=d
-          v=tokenizer([t[1].replace("_"," ") for t in c],add_special_tokens=False)["input_ids"]
-          for i in range(len(v)-1,-1,-1):
-            for j in range(1,len(v[i])):
-              c.insert(i+1,[c[i][0],"_","_","X","_","_",c[i][0],"goeswith","_","_"])
-          y=["0"]+[t[0] for t in c]
-          h=[i if t[6]=="0" else y.index(t[6]) for i,t in enumerate(c,1)]
-          p,v=[t[3]+"|"+t[5]+"|"+t[7] for t in c],sum(v,[])
-          if len(v)<tokenizer.model_max_length-3:
-            self.ids.append([cls]+v+[sep])
-            self.tags.append([dep]+p+[dep])
-            label=set(sum([self.tags[-1],list(label)],[]))
-            for i,k in enumerate(v):
-              self.ids.append([cls]+v[0:i]+[msk]+v[i+1:]+[sep,k])
-              self.tags.append([dep]+[t if h[j]==i+1 else dep for j,t in enumerate(p)]+[dep,dep])
+          for x in [lambda i:i.replace(" ","_"),lambda i:i.replace("_"," ")]:
+            d=list(c)
+            v=tokenizer([x(t[1]) for t in d],add_special_tokens=False)["input_ids"]
+            for i in range(len(v)-1,-1,-1):
+              for j in range(1,len(v[i])):
+                d.insert(i+1,[d[i][0],"_","_","X","_","_",d[i][0],"goeswith","_","_"])
+            y=["0"]+[t[0] for t in d]
+            h=[i if t[6]=="0" else y.index(t[6]) for i,t in enumerate(d,1)]
+            p,v=[t[3]+"|"+t[5]+"|"+t[7] for t in d],sum(v,[])
+            if len(v)<tokenizer.model_max_length-3:
+              self.ids.append([cls]+v+[sep])
+              self.tags.append([dep]+p+[dep])
+              label=set(sum([self.tags[-1],list(label)],[]))
+              for i,k in enumerate(v):
+                self.ids.append([cls]+v[0:i]+[msk]+v[i+1:]+[sep,k])
+                self.tags.append([dep]+[t if h[j]==i+1 else dep for j,t in enumerate(p)]+[dep,dep])
           c=[]
     self.label2id={l:i for i,l in enumerate(sorted(label))}
   def __call__(*args):