KoichiYasuoka
commited on
Commit
•
dfe0ec5
1
Parent(s):
758e4ec
tentative lemma support
Browse files
ud.py
CHANGED
@@ -25,7 +25,10 @@ class UniversalDependenciesPipeline(TokenClassificationPipeline):
|
|
25 |
h=ufal.chu_liu_edmonds.chu_liu_edmonds(m)[0]
|
26 |
v=[(s,e) for s,e in model_output["offset_mapping"][0].tolist() if s<e]
|
27 |
q=[self.model.config.id2label[p[i,j]].split("|") for i,j in enumerate(h)]
|
28 |
-
|
|
|
|
|
|
|
29 |
for i,j in reversed(list(enumerate(q[2:],2))):
|
30 |
if j[-1]=="goeswith":
|
31 |
h=[b if i>b else b-1 for a,b in enumerate(h) if i!=a]
|
@@ -34,5 +37,5 @@ class UniversalDependenciesPipeline(TokenClassificationPipeline):
|
|
34 |
t=model_output["sentence"].replace("\n"," ")
|
35 |
u="# text = "+t+"\n"
|
36 |
for i,(s,e) in enumerate(v,1):
|
37 |
-
u+="\t".join([str(i),t[s:e],"_",q[i][0],"_","|".join(q[i][1:-1]),str(h[i]),q[i][-1],"_","_" if i<len(v) and e<v[i][0] else "SpaceAfter=No"])+"\n"
|
38 |
return u+"\n"
|
|
|
25 |
h=ufal.chu_liu_edmonds.chu_liu_edmonds(m)[0]
|
26 |
v=[(s,e) for s,e in model_output["offset_mapping"][0].tolist() if s<e]
|
27 |
q=[self.model.config.id2label[p[i,j]].split("|") for i,j in enumerate(h)]
|
28 |
+
g=False
|
29 |
+
if "aggregation_strategy" in kwargs:
|
30 |
+
g=kwargs["aggregation_strategy"]!="none"
|
31 |
+
if g:
|
32 |
for i,j in reversed(list(enumerate(q[2:],2))):
|
33 |
if j[-1]=="goeswith":
|
34 |
h=[b if i>b else b-1 for a,b in enumerate(h) if i!=a]
|
|
|
37 |
t=model_output["sentence"].replace("\n"," ")
|
38 |
u="# text = "+t+"\n"
|
39 |
for i,(s,e) in enumerate(v,1):
|
40 |
+
u+="\t".join([str(i),t[s:e],t[s:e] if g else "_",q[i][0],"_","|".join(q[i][1:-1]),str(h[i]),q[i][-1],"_","_" if i<len(v) and e<v[i][0] else "SpaceAfter=No"])+"\n"
|
41 |
return u+"\n"
|