model update

README.md

@@ -14,7 +14,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.7073412698412699
+      value: 0.8037103174603175
   - task:
       name: Analogy Questions (SAT full)
       type: multiple-choice-qa
@@ -25,7 +25,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.4090909090909091
+      value: 0.4679144385026738
   - task:
       name: Analogy Questions (SAT)
       type: multiple-choice-qa
@@ -36,7 +36,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.41543026706231456
+      value: 0.4836795252225519
   - task:
       name: Analogy Questions (BATS)
       type: multiple-choice-qa
@@ -47,7 +47,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.5186214563646471
+      value: 0.5102834908282379
   - task:
       name: Analogy Questions (Google)
       type: multiple-choice-qa
@@ -58,7 +58,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.736
+      value: 0.75
   - task:
       name: Analogy Questions (U2)
       type: multiple-choice-qa
@@ -69,7 +69,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.37719298245614036
+      value: 0.42543859649122806
   - task:
       name: Analogy Questions (U4)
       type: multiple-choice-qa
@@ -80,7 +80,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.41435185185185186
+      value: 0.4398148148148148
   - task:
       name: Analogy Questions (ConceptNet Analogy)
       type: multiple-choice-qa
@@ -91,7 +91,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.13926174496644295
+      value: 0.18791946308724833
   - task:
       name: Analogy Questions (TREX Analogy)
       type: multiple-choice-qa
@@ -102,7 +102,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.6830601092896175
+      value: 0.8360655737704918
   - task:
       name: Analogy Questions (NELL-ONE Analogy)
       type: multiple-choice-qa
@@ -113,7 +113,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.7033333333333334
+      value: 0.7216666666666667
   - task:
       name: Lexical Relation Classification (BLESS)
       type: classification
@@ -124,10 +124,10 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: 0.9050775952990809
+      value: 0.893174627090553
     - name: F1 (macro)
       type: f1_macro
-      value: 0.9018746017633608
+      value: 0.8930103376872522
   - task:
       name: Lexical Relation Classification (CogALexV)
       type: classification
@@ -138,10 +138,10 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: 0.8086854460093896
+      value: 0.828169014084507
     - name: F1 (macro)
       type: f1_macro
-      value: 0.5821540718441495
+      value: 0.635622257984698
   - task:
       name: Lexical Relation Classification (EVALution)
       type: classification
@@ -152,10 +152,10 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: 0.6256771397616468
+      value: 0.6473456121343445
     - name: F1 (macro)
       type: f1_macro
-      value: 0.6267164366528899
+      value: 0.6272978919061384
   - task:
       name: Lexical Relation Classification (K&H+N)
       type: classification
@@ -166,10 +166,10 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: 0.952145788412047
+      value: 0.9520762328719482
     - name: F1 (macro)
       type: f1_macro
-      value: 0.8707930332216822
+      value: 0.8665439837723805
   - task:
       name: Lexical Relation Classification (ROOT09)
       type: classification
@@ -180,10 +180,10 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: 0.8790347853337511
+      value: 0.8824819805703541
     - name: F1 (macro)
       type: f1_macro
-      value: 0.8753206261467538
+      value: 0.8753300511142968
 
 ---
 # relbert/relbert-roberta-base-nce-t-rex
@@ -191,23 +191,23 @@ model-index:
 RelBERT based on [roberta-base](https://huggingface.co/roberta-base) fine-tuned on [relbert/t_rex_relational_similarity](https://huggingface.co/datasets/relbert/t_rex_relational_similarity) (see the [`relbert`](https://github.com/asahi417/relbert) for more detail of fine-tuning).
 This model achieves the following results on the relation understanding tasks:
 - Analogy Question ([dataset](https://huggingface.co/datasets/relbert/analogy_questions), [full result](https://huggingface.co/relbert/relbert-roberta-base-nce-t-rex/raw/main/analogy.forward.json)):
-    - Accuracy on SAT (full): 0.4090909090909091 
-    - Accuracy on SAT: 0.41543026706231456
-    - Accuracy on BATS: 0.5186214563646471
-    - Accuracy on U2: 0.37719298245614036
-    - Accuracy on U4: 0.41435185185185186
-    - Accuracy on Google: 0.736
-    - Accuracy on ConceptNet Analogy: 0.13926174496644295
-    - Accuracy on T-Rex Analogy: 0.6830601092896175
-    - Accuracy on NELL-ONE Analogy: 0.7033333333333334
+    - Accuracy on SAT (full): 0.4679144385026738 
+    - Accuracy on SAT: 0.4836795252225519
+    - Accuracy on BATS: 0.5102834908282379
+    - Accuracy on U2: 0.42543859649122806
+    - Accuracy on U4: 0.4398148148148148
+    - Accuracy on Google: 0.75
+    - Accuracy on ConceptNet Analogy: 0.18791946308724833
+    - Accuracy on T-Rex Analogy: 0.8360655737704918
+    - Accuracy on NELL-ONE Analogy: 0.7216666666666667
 - Lexical Relation Classification ([dataset](https://huggingface.co/datasets/relbert/lexical_relation_classification), [full result](https://huggingface.co/relbert/relbert-roberta-base-nce-t-rex/raw/main/classification.json)):
-    - Micro F1 score on BLESS: 0.9050775952990809
-    - Micro F1 score on CogALexV: 0.8086854460093896
-    - Micro F1 score on EVALution: 0.6256771397616468
-    - Micro F1 score on K&H+N: 0.952145788412047
-    - Micro F1 score on ROOT09: 0.8790347853337511
+    - Micro F1 score on BLESS: 0.893174627090553
+    - Micro F1 score on CogALexV: 0.828169014084507
+    - Micro F1 score on EVALution: 0.6473456121343445
+    - Micro F1 score on K&H+N: 0.9520762328719482
+    - Micro F1 score on ROOT09: 0.8824819805703541
 - Relation Mapping ([dataset](https://huggingface.co/datasets/relbert/relation_mapping), [full result](https://huggingface.co/relbert/relbert-roberta-base-nce-t-rex/raw/main/relation_mapping.json)):
-    - Accuracy on Relation Mapping: 0.7073412698412699 
+    - Accuracy on Relation Mapping: 0.8037103174603175 
 
 
 ### Usage
@@ -249,7 +249,7 @@ If you use any resource from RelBERT, please consider to cite our [paper](https:
 
 ```
 
-@inproceedings{ushio-etal-2021istilling,
+@inproceedings{ushio-etal-2021-distilling,
     title = "Distilling Relation Embeddings from Pretrained Language Models",
     author = "Ushio, Asahi  and
       Camacho-Collados, Jose  and

analogy.bidirection.json

@@ -1 +1 @@
-{"scan/test": 0.17202970297029702, "sat_full/test": 0.45454545454545453, "sat/test": 0.4688427299703264, "u2/test": 0.3991228070175439, "u4/test": 0.42824074074074076, "google/test": 0.756, "bats/test": 0.5025013896609227, "t_rex_relational_similarity/test": 0.7759562841530054, "conceptnet_relational_similarity/test": 0.1552013422818792, "nell_relational_similarity/test": 0.74, "scan/validation": 0.1853932584269663, "sat/validation": 0.32432432432432434, "u2/validation": 0.3333333333333333, "u4/validation": 0.4375, "google/validation": 0.8, "bats/validation": 0.592964824120603, "semeval2012_relational_similarity/validation": 0.4936708860759494, "t_rex_relational_similarity/validation": 0.38306451612903225, "conceptnet_relational_similarity/validation": 0.11241007194244604, "nell_relational_similarity/validation": 0.61}
+{"scan/test": 0.21349009900990099, "sat_full/test": 0.47058823529411764, "sat/test": 0.49258160237388726, "u2/test": 0.45614035087719296, "u4/test": 0.4722222222222222, "google/test": 0.762, "bats/test": 0.5102834908282379, "t_rex_relational_similarity/test": 0.907103825136612, "conceptnet_relational_similarity/test": 0.1912751677852349, "nell_relational_similarity/test": 0.7516666666666667, "scan/validation": 0.20786516853932585, "sat/validation": 0.2702702702702703, "u2/validation": 0.5, "u4/validation": 0.4583333333333333, "google/validation": 0.76, "bats/validation": 0.5829145728643216, "semeval2012_relational_similarity/validation": 0.5822784810126582, "t_rex_relational_similarity/validation": 0.3689516129032258, "conceptnet_relational_similarity/validation": 0.1303956834532374, "nell_relational_similarity/validation": 0.6275}

analogy.forward.json

@@ -1 +1 @@
-{"t_rex_relational_similarity/validation": 0.4112903225806452, "scan/test": 0.16336633663366337, "sat_full/test": 0.4090909090909091, "sat/test": 0.41543026706231456, "u2/test": 0.37719298245614036, "u4/test": 0.41435185185185186, "google/test": 0.736, "bats/test": 0.5186214563646471, "t_rex_relational_similarity/test": 0.6830601092896175, "conceptnet_relational_similarity/test": 0.13926174496644295, "nell_relational_similarity/test": 0.7033333333333334, "scan/validation": 0.1797752808988764, "sat/validation": 0.35135135135135137, "u2/validation": 0.25, "u4/validation": 0.4166666666666667, "google/validation": 0.78, "bats/validation": 0.6130653266331658, "semeval2012_relational_similarity/validation": 0.5063291139240507, "conceptnet_relational_similarity/validation": 0.1079136690647482, "nell_relational_similarity/validation": 0.5775}
+{"t_rex_relational_similarity/validation": 0.3588709677419355, "scan/test": 0.19616336633663367, "sat_full/test": 0.4679144385026738, "sat/test": 0.4836795252225519, "u2/test": 0.42543859649122806, "u4/test": 0.4398148148148148, "google/test": 0.75, "bats/test": 0.5102834908282379, "t_rex_relational_similarity/test": 0.8360655737704918, "conceptnet_relational_similarity/test": 0.18791946308724833, "nell_relational_similarity/test": 0.7216666666666667, "scan/validation": 0.1853932584269663, "sat/validation": 0.32432432432432434, "u2/validation": 0.5, "u4/validation": 0.4583333333333333, "google/validation": 0.76, "bats/validation": 0.5979899497487438, "semeval2012_relational_similarity/validation": 0.5443037974683544, "conceptnet_relational_similarity/validation": 0.1447841726618705, "nell_relational_similarity/validation": 0.6075}

analogy.reverse.json

@@ -1 +1 @@
-{"scan/test": 0.14727722772277227, "sat_full/test": 0.44385026737967914, "sat/test": 0.4540059347181009, "u2/test": 0.41228070175438597, "u4/test": 0.4305555555555556, "google/test": 0.724, "bats/test": 0.4613674263479711, "t_rex_relational_similarity/test": 0.7704918032786885, "conceptnet_relational_similarity/test": 0.13506711409395974, "nell_relational_similarity/test": 0.74, "scan/validation": 0.15730337078651685, "sat/validation": 0.35135135135135137, "u2/validation": 0.2916666666666667, "u4/validation": 0.4375, "google/validation": 0.78, "bats/validation": 0.5376884422110553, "semeval2012_relational_similarity/validation": 0.4810126582278481, "t_rex_relational_similarity/validation": 0.28225806451612906, "conceptnet_relational_similarity/validation": 0.08992805755395683, "nell_relational_similarity/validation": 0.625}
+{"scan/test": 0.18935643564356436, "sat_full/test": 0.45187165775401067, "sat/test": 0.4807121661721068, "u2/test": 0.4692982456140351, "u4/test": 0.49074074074074076, "google/test": 0.734, "bats/test": 0.47915508615897723, "t_rex_relational_similarity/test": 0.8579234972677595, "conceptnet_relational_similarity/test": 0.16526845637583892, "nell_relational_similarity/test": 0.7316666666666667, "scan/validation": 0.17415730337078653, "sat/validation": 0.1891891891891892, "u2/validation": 0.4166666666666667, "u4/validation": 0.4583333333333333, "google/validation": 0.72, "bats/validation": 0.5326633165829145, "semeval2012_relational_similarity/validation": 0.5822784810126582, "t_rex_relational_similarity/validation": 0.3125, "conceptnet_relational_similarity/validation": 0.10971223021582734, "nell_relational_similarity/validation": 0.6125}

classification.json

@@ -1 +1 @@
-{"lexical_relation_classification/BLESS": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.9050775952990809, "test/f1_macro": 0.9018746017633608, "test/f1_micro": 0.9050775952990809, "test/p_macro": 0.8964215102265775, "test/p_micro": 0.9050775952990809, "test/r_macro": 0.908193811489784, "test/r_micro": 0.9050775952990809}, "lexical_relation_classification/CogALexV": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.8086854460093896, "test/f1_macro": 0.5821540718441495, "test/f1_micro": 0.8086854460093896, "test/p_macro": 0.6193510739469049, "test/p_micro": 0.8086854460093896, "test/r_macro": 0.5532329729031553, "test/r_micro": 0.8086854460093896}, "lexical_relation_classification/EVALution": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.6256771397616468, "test/f1_macro": 0.6267164366528899, "test/f1_micro": 0.6256771397616468, "test/p_macro": 0.6458907318056867, "test/p_micro": 0.6256771397616468, "test/r_macro": 0.6145961528356455, "test/r_micro": 0.6256771397616468}, "lexical_relation_classification/K&H+N": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.952145788412047, "test/f1_macro": 0.8707930332216822, "test/f1_micro": 0.952145788412047, "test/p_macro": 0.8859213667858965, "test/p_micro": 0.952145788412047, "test/r_macro": 0.8572030983309535, "test/r_micro": 0.952145788412047}, "lexical_relation_classification/ROOT09": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.8790347853337511, "test/f1_macro": 0.8753206261467538, "test/f1_micro": 0.8790347853337511, "test/p_macro": 0.8826480235296165, "test/p_micro": 0.8790347853337511, "test/r_macro": 0.8697183763180335, "test/r_micro": 0.8790347853337511}}
+{"lexical_relation_classification/BLESS": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.893174627090553, "test/f1_macro": 0.8930103376872522, "test/f1_micro": 0.893174627090553, "test/p_macro": 0.8769667616715443, "test/p_micro": 0.893174627090553, "test/r_macro": 0.9134767381633374, "test/r_micro": 0.893174627090553, "test/f1/attri": 0.9094736842105263, "test/p/attri": 0.8888888888888888, "test/r/attri": 0.9310344827586207, "test/f1/coord": 0.9519071310116086, "test/p/coord": 0.9288025889967637, "test/r/coord": 0.9761904761904762, "test/f1/event": 0.8120740019474197, "test/p/event": 0.7588717015468608, "test/r/event": 0.8732984293193717, "test/f1/hyper": 0.932761087267525, "test/p/hyper": 0.9340974212034384, "test/r/hyper": 0.9314285714285714, "test/f1/mero": 0.8426270136307311, "test/p/mero": 0.783410138248848, "test/r/mero": 0.9115281501340483, "test/f1/random": 0.9092191080557024, "test/p/random": 0.9677298311444653, "test/r/random": 0.8573803191489362}, "lexical_relation_classification/CogALexV": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.828169014084507, "test/f1_macro": 0.635622257984698, "test/f1_micro": 0.828169014084507, "test/p_macro": 0.6533963907120769, "test/p_micro": 0.828169014084507, "test/r_macro": 0.6223812518853498, "test/r_micro": 0.828169014084507, "test/f1/ANT": 0.6611570247933883, "test/p/ANT": 0.6557377049180327, "test/r/ANT": 0.6666666666666666, "test/f1/HYPER": 0.5735512630014858, "test/p/HYPER": 0.6632302405498282, "test/r/HYPER": 0.5052356020942408, "test/f1/PART_OF": 0.6227272727272727, "test/p/PART_OF": 0.6342592592592593, "test/r/PART_OF": 0.6116071428571429, "test/f1/RANDOM": 0.9215434083601286, "test/p/RANDOM": 0.9066751028155647, "test/r/RANDOM": 0.9369074861065708, "test/f1/SYN": 0.3991323210412147, "test/p/SYN": 0.40707964601769914, "test/r/SYN": 0.39148936170212767}, "lexical_relation_classification/EVALution": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.6473456121343445, "test/f1_macro": 0.6272978919061384, "test/f1_micro": 0.6473456121343445, "test/p_macro": 0.6379993388182703, "test/p_micro": 0.6473456121343445, "test/r_macro": 0.6283944073527691, "test/r_micro": 0.6473456121343445, "test/f1/Antonym": 0.745049504950495, "test/p/Antonym": 0.7659033078880407, "test/r/Antonym": 0.7253012048192771, "test/f1/HasA": 0.6987179487179488, "test/p/HasA": 0.6411764705882353, "test/r/HasA": 0.7676056338028169, "test/f1/HasProperty": 0.8115501519756838, "test/p/HasProperty": 0.7946428571428571, "test/r/HasProperty": 0.8291925465838509, "test/f1/IsA": 0.6021505376344087, "test/p/IsA": 0.5460992907801419, "test/r/IsA": 0.6710239651416122, "test/f1/MadeOf": 0.543046357615894, "test/p/MadeOf": 0.6307692307692307, "test/r/MadeOf": 0.47674418604651164, "test/f1/PartOf": 0.671280276816609, "test/p/PartOf": 0.6736111111111112, "test/r/PartOf": 0.6689655172413793, "test/f1/Synonym": 0.31929046563192903, "test/p/Synonym": 0.41379310344827586, "test/r/Synonym": 0.259927797833935}, "lexical_relation_classification/K&H+N": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.9520762328719482, "test/f1_macro": 0.8665439837723805, "test/f1_micro": 0.9520762328719482, "test/p_macro": 0.883226419598165, "test/p_micro": 0.9520762328719482, "test/r_macro": 0.8522903227547688, "test/r_micro": 0.9520762328719482, "test/f1/false": 0.9600705934259873, "test/p/false": 0.9525755143732672, "test/r/false": 0.967684553809665, "test/f1/hypo": 0.9270428015564203, "test/p/hypo": 0.9398422090729783, "test/r/hypo": 0.9145873320537428, "test/f1/mero": 0.6199095022624435, "test/p/mero": 0.6782178217821783, "test/r/mero": 0.5708333333333333, "test/f1/sibl": 0.9591530378446709, "test/p/sibl": 0.9622701331642359, "test/r/sibl": 0.9560560718223342}, "lexical_relation_classification/ROOT09": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.8824819805703541, "test/f1_macro": 0.8753300511142968, "test/f1_micro": 0.8824819805703541, "test/p_macro": 0.8890447043219271, "test/p_micro": 0.8824819805703541, "test/r_macro": 0.867841782002778, "test/r_micro": 0.8824819805703541, "test/f1/COORD": 0.9745145631067961, "test/p/COORD": 0.9651442307692307, "test/r/COORD": 0.9840686274509803, "test/f1/HYPER": 0.7605442176870748, "test/p/HYPER": 0.8456883509833586, "test/r/HYPER": 0.6909765142150803, "test/f1/RANDOM": 0.8909313725490197, "test/p/RANDOM": 0.856301531213192, "test/r/RANDOM": 0.9284802043422733}}

config.json

@@ -21,7 +21,7 @@
   "position_embedding_type": "absolute",
   "relbert_config": {
     "aggregation_mode": "average_no_mask",
-    "template": "I wasn\u2019t aware of this relationship, but I just read in the encyclopedia that <subj> is the <mask> of <obj>"
+    "template": "Today, I finally discovered the relation between <subj> and <obj> : <subj> is the <mask> of <obj>"
   },
   "torch_dtype": "float32",
   "transformers_version": "4.26.1",

finetuning_config.json

@@ -1,5 +1,5 @@
 {
-  "template": "I wasn\u2019t aware of this relationship, but I just read in the encyclopedia that <subj> is the <mask> of <obj>",
+  "template": "Today, I finally discovered the relation between <subj> and <obj> : <subj> is the <mask> of <obj>",
   "model": "roberta-base",
   "max_length": 64,
   "epoch": 5,

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:83859baf67c8581db24f6868342cf046a6c771af90428be8e36e5c745ee96f61
-size 498652017
+oid sha256:36e5182ebc888d5e917f1ef86fc498b0177733c088e40a417399edb87ab24e3f
+size 498653741

tokenizer.json

@@ -53,7 +53,8 @@
   "pre_tokenizer": {
     "type": "ByteLevel",
     "add_prefix_space": false,
-    "trim_offsets": true
+    "trim_offsets": true,
+    "use_regex": true
   },
   "post_processor": {
     "type": "RobertaProcessing",
@@ -71,7 +72,8 @@
   "decoder": {
     "type": "ByteLevel",
     "add_prefix_space": true,
-    "trim_offsets": true
+    "trim_offsets": true,
+    "use_regex": true
   },
   "model": {
     "type": "BPE",