Hugging Face's logo

bwang0911
/
jev2-legal

Sentence Similarity
sentence-transformers
Safetensors
English
mpnet
feature-extraction
Generated from Trainer
dataset_size:53224
loss:MultipleNegativesRankingLoss
Eval Results
Inference Endpoints
Model card Files Community
bwang0911 commited on
Commit
bdb5e9c
·
verified ·
1 Parent(s): 9a7444d

Add new SentenceTransformer model

Browse files
Files changed (2) hide show
  1. README.md +37 -37
  2. model.safetensors +1 -1
README.md CHANGED
@@ -390,7 +390,7 @@ model-index:
390
  value: 0.36276984126984124
391
  name: Cosine Mrr@10
392
  - type: cosine_map@100
393
- value: 0.23063570433429445
394
  name: Cosine Map@100
395
  - task:
396
  type: information-retrieval
@@ -421,7 +421,7 @@ model-index:
421
  value: 0.132
422
  name: Cosine Precision@5
423
  - type: cosine_precision@10
424
- value: 0.10800000000000001
425
  name: Cosine Precision@10
426
  - type: cosine_recall@1
427
  value: 0.06499999999999999
@@ -433,16 +433,16 @@ model-index:
433
  value: 0.16399999999999998
434
  name: Cosine Recall@5
435
  - type: cosine_recall@10
436
- value: 0.256
437
  name: Cosine Recall@10
438
  - type: cosine_ndcg@10
439
- value: 0.22688341584203112
440
  name: Cosine Ndcg@10
441
  - type: cosine_mrr@10
442
  value: 0.3726904761904762
443
  name: Cosine Mrr@10
444
  - type: cosine_map@100
445
- value: 0.19663910082742547
446
  name: Cosine Map@100
447
  - task:
448
  type: information-retrieval
@@ -507,7 +507,7 @@ model-index:
507
  value: 0.7382352941176471
508
  name: Cosine Accuracy@1
509
  - type: cosine_accuracy@3
510
- value: 0.8941176470588236
511
  name: Cosine Accuracy@3
512
  - type: cosine_accuracy@5
513
  value: 0.9294117647058824
@@ -519,7 +519,7 @@ model-index:
519
  value: 0.7382352941176471
520
  name: Cosine Precision@1
521
  - type: cosine_precision@3
522
- value: 0.2980392156862745
523
  name: Cosine Precision@3
524
  - type: cosine_precision@5
525
  value: 0.18588235294117644
@@ -531,7 +531,7 @@ model-index:
531
  value: 0.7382352941176471
532
  name: Cosine Recall@1
533
  - type: cosine_recall@3
534
- value: 0.8941176470588236
535
  name: Cosine Recall@3
536
  - type: cosine_recall@5
537
  value: 0.9294117647058824
@@ -540,13 +540,13 @@ model-index:
540
  value: 0.9647058823529412
541
  name: Cosine Recall@10
542
  - type: cosine_ndcg@10
543
- value: 0.8568617986155895
544
  name: Cosine Ndcg@10
545
  - type: cosine_mrr@10
546
- value: 0.8217670401493932
547
  name: Cosine Mrr@10
548
  - type: cosine_map@100
549
- value: 0.822962290292461
550
  name: Cosine Map@100
551
  - task:
552
  type: information-retrieval
@@ -592,13 +592,13 @@ model-index:
592
  value: 0.7263897780623133
593
  name: Cosine Recall@10
594
  - type: cosine_ndcg@10
595
- value: 0.5978698544115026
596
  name: Cosine Ndcg@10
597
  - type: cosine_mrr@10
598
- value: 0.5853118712273642
599
  name: Cosine Mrr@10
600
  - type: cosine_map@100
601
- value: 0.555963840687379
602
  name: Cosine Map@100
603
  ---
604
 
@@ -710,20 +710,20 @@ You can finetune this model on your own dataset.
710
  | Metric | mteb/AILA_casedocs | mteb/AILA_statutes | mteb/legalbench_consumer_contracts_qa | mteb/legalbench_corporate_lobbying | mteb/legal_summarization |
711
  |:--------------------|:-------------------|:-------------------|:--------------------------------------|:-----------------------------------|:-------------------------|
712
  | cosine_accuracy@1 | 0.32 | 0.26 | 0.452 | 0.7382 | 0.493 |
713
- | cosine_accuracy@3 | 0.36 | 0.42 | 0.6843 | 0.8941 | 0.6479 |
714
  | cosine_accuracy@5 | 0.38 | 0.52 | 0.7879 | 0.9294 | 0.7113 |
715
  | cosine_accuracy@10 | 0.54 | 0.68 | 0.8662 | 0.9647 | 0.7958 |
716
  | cosine_precision@1 | 0.32 | 0.26 | 0.452 | 0.7382 | 0.493 |
717
- | cosine_precision@3 | 0.2 | 0.16 | 0.2281 | 0.298 | 0.2371 |
718
  | cosine_precision@5 | 0.14 | 0.132 | 0.1576 | 0.1859 | 0.1641 |
719
- | cosine_precision@10 | 0.106 | 0.108 | 0.0866 | 0.0965 | 0.0986 |
720
  | cosine_recall@1 | 0.1001 | 0.065 | 0.452 | 0.7382 | 0.4376 |
721
- | cosine_recall@3 | 0.174 | 0.125 | 0.6843 | 0.8941 | 0.5706 |
722
  | cosine_recall@5 | 0.2052 | 0.164 | 0.7879 | 0.9294 | 0.6356 |
723
- | cosine_recall@10 | 0.3084 | 0.256 | 0.8662 | 0.9647 | 0.7264 |
724
- | **cosine_ndcg@10** | **0.2682** | **0.2269** | **0.6554** | **0.8569** | **0.5979** |
725
- | cosine_mrr@10 | 0.3628 | 0.3727 | 0.5882 | 0.8218 | 0.5853 |
726
- | cosine_map@100 | 0.2306 | 0.1966 | 0.5945 | 0.823 | 0.556 |
727
 
728
  <!--
729
  ## Bias, Risks and Limitations
@@ -1067,22 +1067,22 @@ You can finetune this model on your own dataset.
1067
  | Epoch | Step | Training Loss | mteb/AILA_casedocs_cosine_ndcg@10 | mteb/AILA_statutes_cosine_ndcg@10 | mteb/legalbench_consumer_contracts_qa_cosine_ndcg@10 | mteb/legalbench_corporate_lobbying_cosine_ndcg@10 | mteb/legal_summarization_cosine_ndcg@10 |
1068
  |:------:|:----:|:-------------:|:---------------------------------:|:---------------------------------:|:----------------------------------------------------:|:-------------------------------------------------:|:---------------------------------------:|
1069
  | 0 | 0 | - | 0.1972 | 0.2052 | 0.6560 | 0.8641 | 0.5900 |
1070
- | 0.1196 | 100 | - | 0.1936 | 0.2103 | 0.6553 | 0.8662 | 0.5969 |
1071
- | 0.2392 | 200 | - | 0.2130 | 0.2135 | 0.6610 | 0.8652 | 0.5988 |
1072
- | 0.3589 | 300 | - | 0.2148 | 0.2145 | 0.6626 | 0.8651 | 0.6051 |
1073
  | 0.4785 | 400 | - | 0.2227 | 0.2089 | 0.6584 | 0.8618 | 0.6050 |
1074
- | 0.5981 | 500 | 4.8514 | 0.2253 | 0.2046 | 0.6583 | 0.8558 | 0.6033 |
1075
- | 0.7177 | 600 | - | 0.2301 | 0.2104 | 0.6585 | 0.8568 | 0.6003 |
1076
- | 0.8373 | 700 | - | 0.2421 | 0.2037 | 0.6588 | 0.8592 | 0.6007 |
1077
- | 0.9569 | 800 | - | 0.2560 | 0.2089 | 0.6589 | 0.8597 | 0.6013 |
1078
- | 1.0766 | 900 | - | 0.2518 | 0.2122 | 0.6592 | 0.8616 | 0.6014 |
1079
- | 1.1962 | 1000 | 3.2784 | 0.2638 | 0.2193 | 0.6598 | 0.8622 | 0.6010 |
1080
- | 1.3158 | 1100 | - | 0.2660 | 0.2193 | 0.6601 | 0.8630 | 0.6008 |
1081
- | 1.4354 | 1200 | - | 0.2617 | 0.2237 | 0.6594 | 0.8601 | 0.5990 |
1082
- | 1.5550 | 1300 | - | 0.2656 | 0.2249 | 0.6571 | 0.8563 | 0.5990 |
1083
- | 1.6746 | 1400 | - | 0.2648 | 0.2254 | 0.6568 | 0.8558 | 0.5992 |
1084
- | 1.7943 | 1500 | 3.3186 | 0.2683 | 0.2283 | 0.6564 | 0.8557 | 0.5988 |
1085
- | 1.9139 | 1600 | - | 0.2682 | 0.2269 | 0.6554 | 0.8569 | 0.5979 |
1086
 
1087
 
1088
  ### Framework Versions
 
390
  value: 0.36276984126984124
391
  name: Cosine Mrr@10
392
  - type: cosine_map@100
393
+ value: 0.23063036312378082
394
  name: Cosine Map@100
395
  - task:
396
  type: information-retrieval
 
421
  value: 0.132
422
  name: Cosine Precision@5
423
  - type: cosine_precision@10
424
+ value: 0.11
425
  name: Cosine Precision@10
426
  - type: cosine_recall@1
427
  value: 0.06499999999999999
 
433
  value: 0.16399999999999998
434
  name: Cosine Recall@5
435
  - type: cosine_recall@10
436
+ value: 0.261
437
  name: Cosine Recall@10
438
  - type: cosine_ndcg@10
439
+ value: 0.22914031866900986
440
  name: Cosine Ndcg@10
441
  - type: cosine_mrr@10
442
  value: 0.3726904761904762
443
  name: Cosine Mrr@10
444
  - type: cosine_map@100
445
+ value: 0.19673834325166786
446
  name: Cosine Map@100
447
  - task:
448
  type: information-retrieval
 
507
  value: 0.7382352941176471
508
  name: Cosine Accuracy@1
509
  - type: cosine_accuracy@3
510
+ value: 0.8970588235294118
511
  name: Cosine Accuracy@3
512
  - type: cosine_accuracy@5
513
  value: 0.9294117647058824
 
519
  value: 0.7382352941176471
520
  name: Cosine Precision@1
521
  - type: cosine_precision@3
522
+ value: 0.29901960784313725
523
  name: Cosine Precision@3
524
  - type: cosine_precision@5
525
  value: 0.18588235294117644
 
531
  value: 0.7382352941176471
532
  name: Cosine Recall@1
533
  - type: cosine_recall@3
534
+ value: 0.8970588235294118
535
  name: Cosine Recall@3
536
  - type: cosine_recall@5
537
  value: 0.9294117647058824
 
540
  value: 0.9647058823529412
541
  name: Cosine Recall@10
542
  - type: cosine_ndcg@10
543
+ value: 0.8570656910918443
544
  name: Cosine Ndcg@10
545
  - type: cosine_mrr@10
546
+ value: 0.822012138188609
547
  name: Cosine Mrr@10
548
  - type: cosine_map@100
549
+ value: 0.8232073883316766
550
  name: Cosine Map@100
551
  - task:
552
  type: information-retrieval
 
592
  value: 0.7263897780623133
593
  name: Cosine Recall@10
594
  - type: cosine_ndcg@10
595
+ value: 0.5975871811338554
596
  name: Cosine Ndcg@10
597
  - type: cosine_mrr@10
598
+ value: 0.5847250167672702
599
  name: Cosine Mrr@10
600
  - type: cosine_map@100
601
+ value: 0.555669852845899
602
  name: Cosine Map@100
603
  ---
604
 
 
710
  | Metric | mteb/AILA_casedocs | mteb/AILA_statutes | mteb/legalbench_consumer_contracts_qa | mteb/legalbench_corporate_lobbying | mteb/legal_summarization |
711
  |:--------------------|:-------------------|:-------------------|:--------------------------------------|:-----------------------------------|:-------------------------|
712
  | cosine_accuracy@1 | 0.32 | 0.26 | 0.452 | 0.7382 | 0.493 |
713
+ | cosine_accuracy@3 | 0.36 | 0.42 | 0.6843 | 0.8971 | 0.6479 |
714
  | cosine_accuracy@5 | 0.38 | 0.52 | 0.7879 | 0.9294 | 0.7113 |
715
  | cosine_accuracy@10 | 0.54 | 0.68 | 0.8662 | 0.9647 | 0.7958 |
716
  | cosine_precision@1 | 0.32 | 0.26 | 0.452 | 0.7382 | 0.493 |
717
+ | cosine_precision@3 | 0.2 | 0.16 | 0.2281 | 0.299 | 0.2371 |
718
  | cosine_precision@5 | 0.14 | 0.132 | 0.1576 | 0.1859 | 0.1641 |
719
+ | cosine_precision@10 | 0.106 | 0.11 | 0.0866 | 0.0965 | 0.0986 |
720
  | cosine_recall@1 | 0.1001 | 0.065 | 0.452 | 0.7382 | 0.4376 |
721
+ | cosine_recall@3 | 0.174 | 0.125 | 0.6843 | 0.8971 | 0.5706 |
722
  | cosine_recall@5 | 0.2052 | 0.164 | 0.7879 | 0.9294 | 0.6356 |
723
+ | cosine_recall@10 | 0.3084 | 0.261 | 0.8662 | 0.9647 | 0.7264 |
724
+ | **cosine_ndcg@10** | **0.2682** | **0.2291** | **0.6554** | **0.8571** | **0.5976** |
725
+ | cosine_mrr@10 | 0.3628 | 0.3727 | 0.5882 | 0.822 | 0.5847 |
726
+ | cosine_map@100 | 0.2306 | 0.1967 | 0.5945 | 0.8232 | 0.5557 |
727
 
728
  <!--
729
  ## Bias, Risks and Limitations
 
1067
  | Epoch | Step | Training Loss | mteb/AILA_casedocs_cosine_ndcg@10 | mteb/AILA_statutes_cosine_ndcg@10 | mteb/legalbench_consumer_contracts_qa_cosine_ndcg@10 | mteb/legalbench_corporate_lobbying_cosine_ndcg@10 | mteb/legal_summarization_cosine_ndcg@10 |
1068
  |:------:|:----:|:-------------:|:---------------------------------:|:---------------------------------:|:----------------------------------------------------:|:-------------------------------------------------:|:---------------------------------------:|
1069
  | 0 | 0 | - | 0.1972 | 0.2052 | 0.6560 | 0.8641 | 0.5900 |
1070
+ | 0.1196 | 100 | - | 0.1936 | 0.2103 | 0.6552 | 0.8661 | 0.5969 |
1071
+ | 0.2392 | 200 | - | 0.2130 | 0.2112 | 0.6601 | 0.8650 | 0.5988 |
1072
+ | 0.3589 | 300 | - | 0.2148 | 0.2145 | 0.6626 | 0.8650 | 0.6051 |
1073
  | 0.4785 | 400 | - | 0.2227 | 0.2089 | 0.6584 | 0.8618 | 0.6050 |
1074
+ | 0.5981 | 500 | 4.8514 | 0.2253 | 0.2046 | 0.6591 | 0.8562 | 0.6030 |
1075
+ | 0.7177 | 600 | - | 0.2301 | 0.2104 | 0.6587 | 0.8557 | 0.6003 |
1076
+ | 0.8373 | 700 | - | 0.2421 | 0.2037 | 0.6588 | 0.8593 | 0.6007 |
1077
+ | 0.9569 | 800 | - | 0.2560 | 0.2089 | 0.6589 | 0.8599 | 0.6014 |
1078
+ | 1.0766 | 900 | - | 0.2518 | 0.2123 | 0.6589 | 0.8626 | 0.6014 |
1079
+ | 1.1962 | 1000 | 3.2784 | 0.2639 | 0.2214 | 0.6598 | 0.8638 | 0.6010 |
1080
+ | 1.3158 | 1100 | - | 0.2660 | 0.2193 | 0.6602 | 0.8633 | 0.6008 |
1081
+ | 1.4354 | 1200 | - | 0.2652 | 0.2237 | 0.6594 | 0.8601 | 0.5989 |
1082
+ | 1.5550 | 1300 | - | 0.2656 | 0.2249 | 0.6571 | 0.8564 | 0.5990 |
1083
+ | 1.6746 | 1400 | - | 0.2648 | 0.2254 | 0.6558 | 0.8559 | 0.5991 |
1084
+ | 1.7943 | 1500 | 3.3186 | 0.2683 | 0.2288 | 0.6564 | 0.8559 | 0.5995 |
1085
+ | 1.9139 | 1600 | - | 0.2682 | 0.2291 | 0.6554 | 0.8571 | 0.5976 |
1086
 
1087
 
1088
  ### Framework Versions
model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:c6e9b8047dac0268b40a558cc5a9b1d602366a655283a7049668101ace9062ef
3
  size 90864192
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6b5dfee6977cb883265beefa89c6b872bb304459d35e061c03b3502bde6ab500
3
  size 90864192