|
--- |
|
tags: |
|
- merge |
|
- mergekit |
|
- lazymergekit |
|
- HuggingFaceH4/mistral-7b-anthropic |
|
- HuggingFaceH4/mistral-7b-grok |
|
base_model: |
|
- HuggingFaceH4/mistral-7b-anthropic |
|
- HuggingFaceH4/mistral-7b-grok |
|
license: apache-2.0 |
|
--- |
|
|
|
# JOSIE_Beta-8-7B-slerp |
|
|
|
JOSIE_Beta-8-7B-slerp is a merge of the following models using [LazyMergekit](https://colab.research.google.com/drive/1obulZ1ROXHjYLn6PPZJwRR6GzgQogxxb?usp=sharing): |
|
* [HuggingFaceH4/mistral-7b-anthropic](https://huggingface.co/HuggingFaceH4/mistral-7b-anthropic) |
|
* [HuggingFaceH4/mistral-7b-grok](https://huggingface.co/HuggingFaceH4/mistral-7b-grok) |
|
|
|
# Important!!! |
|
|
|
Upon seing the eval benchmarks on the LLM Leaderbard this model performs the worst. the best performing one (in the leaderboard) is the 3 beta version. |
|
|
|
```json |
|
{ |
|
"all": { |
|
"acc": 0.6212846416057433, |
|
"acc_stderr": 0.03289607423593368, |
|
"acc_norm": 0.6268274539918854, |
|
"acc_norm_stderr": 0.03356884635772938, |
|
"mc1": 0.3157894736842105, |
|
"mc1_stderr": 0.016272287957916923, |
|
"mc2": 0.4868797251828956, |
|
"mc2_stderr": 0.01529943410920313 |
|
}, |
|
"harness|arc:challenge|25": { |
|
"acc": 0.5776450511945392, |
|
"acc_stderr": 0.014434138713379981, |
|
"acc_norm": 0.6040955631399317, |
|
"acc_norm_stderr": 0.014291228393536592 |
|
}, |
|
"harness|hellaswag|10": { |
|
"acc": 0.6363274248157738, |
|
"acc_stderr": 0.004800728138792394, |
|
"acc_norm": 0.8365863373829915, |
|
"acc_norm_stderr": 0.0036898701424130753 |
|
}, |
|
"harness|hendrycksTest-abstract_algebra|5": { |
|
"acc": 0.31, |
|
"acc_stderr": 0.046482319871173156, |
|
"acc_norm": 0.31, |
|
"acc_norm_stderr": 0.046482319871173156 |
|
}, |
|
"harness|hendrycksTest-anatomy|5": { |
|
"acc": 0.6, |
|
"acc_stderr": 0.04232073695151589, |
|
"acc_norm": 0.6, |
|
"acc_norm_stderr": 0.04232073695151589 |
|
}, |
|
"harness|hendrycksTest-astronomy|5": { |
|
"acc": 0.5986842105263158, |
|
"acc_stderr": 0.039889037033362836, |
|
"acc_norm": 0.5986842105263158, |
|
"acc_norm_stderr": 0.039889037033362836 |
|
}, |
|
"harness|hendrycksTest-business_ethics|5": { |
|
"acc": 0.58, |
|
"acc_stderr": 0.049604496374885836, |
|
"acc_norm": 0.58, |
|
"acc_norm_stderr": 0.049604496374885836 |
|
}, |
|
"harness|hendrycksTest-clinical_knowledge|5": { |
|
"acc": 0.6867924528301886, |
|
"acc_stderr": 0.028544793319055326, |
|
"acc_norm": 0.6867924528301886, |
|
"acc_norm_stderr": 0.028544793319055326 |
|
}, |
|
"harness|hendrycksTest-college_biology|5": { |
|
"acc": 0.7083333333333334, |
|
"acc_stderr": 0.038009680605548594, |
|
"acc_norm": 0.7083333333333334, |
|
"acc_norm_stderr": 0.038009680605548594 |
|
}, |
|
"harness|hendrycksTest-college_chemistry|5": { |
|
"acc": 0.53, |
|
"acc_stderr": 0.05016135580465919, |
|
"acc_norm": 0.53, |
|
"acc_norm_stderr": 0.05016135580465919 |
|
}, |
|
"harness|hendrycksTest-college_computer_science|5": { |
|
"acc": 0.51, |
|
"acc_stderr": 0.05024183937956912, |
|
"acc_norm": 0.51, |
|
"acc_norm_stderr": 0.05024183937956912 |
|
}, |
|
"harness|hendrycksTest-college_mathematics|5": { |
|
"acc": 0.39, |
|
"acc_stderr": 0.04902071300001975, |
|
"acc_norm": 0.39, |
|
"acc_norm_stderr": 0.04902071300001975 |
|
}, |
|
"harness|hendrycksTest-college_medicine|5": { |
|
"acc": 0.6184971098265896, |
|
"acc_stderr": 0.03703851193099521, |
|
"acc_norm": 0.6184971098265896, |
|
"acc_norm_stderr": 0.03703851193099521 |
|
}, |
|
"harness|hendrycksTest-college_physics|5": { |
|
"acc": 0.39215686274509803, |
|
"acc_stderr": 0.048580835742663454, |
|
"acc_norm": 0.39215686274509803, |
|
"acc_norm_stderr": 0.048580835742663454 |
|
}, |
|
"harness|hendrycksTest-computer_security|5": { |
|
"acc": 0.75, |
|
"acc_stderr": 0.04351941398892446, |
|
"acc_norm": 0.75, |
|
"acc_norm_stderr": 0.04351941398892446 |
|
}, |
|
"harness|hendrycksTest-conceptual_physics|5": { |
|
"acc": 0.5659574468085107, |
|
"acc_stderr": 0.03240038086792747, |
|
"acc_norm": 0.5659574468085107, |
|
"acc_norm_stderr": 0.03240038086792747 |
|
}, |
|
"harness|hendrycksTest-econometrics|5": { |
|
"acc": 0.47368421052631576, |
|
"acc_stderr": 0.04697085136647863, |
|
"acc_norm": 0.47368421052631576, |
|
"acc_norm_stderr": 0.04697085136647863 |
|
}, |
|
"harness|hendrycksTest-electrical_engineering|5": { |
|
"acc": 0.5586206896551724, |
|
"acc_stderr": 0.04137931034482757, |
|
"acc_norm": 0.5586206896551724, |
|
"acc_norm_stderr": 0.04137931034482757 |
|
}, |
|
"harness|hendrycksTest-elementary_mathematics|5": { |
|
"acc": 0.41005291005291006, |
|
"acc_stderr": 0.025331202438944437, |
|
"acc_norm": 0.41005291005291006, |
|
"acc_norm_stderr": 0.025331202438944437 |
|
}, |
|
"harness|hendrycksTest-formal_logic|5": { |
|
"acc": 0.3888888888888889, |
|
"acc_stderr": 0.04360314860077459, |
|
"acc_norm": 0.3888888888888889, |
|
"acc_norm_stderr": 0.04360314860077459 |
|
}, |
|
"harness|hendrycksTest-global_facts|5": { |
|
"acc": 0.4, |
|
"acc_stderr": 0.049236596391733084, |
|
"acc_norm": 0.4, |
|
"acc_norm_stderr": 0.049236596391733084 |
|
}, |
|
"harness|hendrycksTest-high_school_biology|5": { |
|
"acc": 0.7580645161290323, |
|
"acc_stderr": 0.024362599693031083, |
|
"acc_norm": 0.7580645161290323, |
|
"acc_norm_stderr": 0.024362599693031083 |
|
}, |
|
"harness|hendrycksTest-high_school_chemistry|5": { |
|
"acc": 0.5221674876847291, |
|
"acc_stderr": 0.03514528562175008, |
|
"acc_norm": 0.5221674876847291, |
|
"acc_norm_stderr": 0.03514528562175008 |
|
}, |
|
"harness|hendrycksTest-high_school_computer_science|5": { |
|
"acc": 0.63, |
|
"acc_stderr": 0.04852365870939099, |
|
"acc_norm": 0.63, |
|
"acc_norm_stderr": 0.04852365870939099 |
|
}, |
|
"harness|hendrycksTest-high_school_european_history|5": { |
|
"acc": 0.7515151515151515, |
|
"acc_stderr": 0.033744026441394036, |
|
"acc_norm": 0.7515151515151515, |
|
"acc_norm_stderr": 0.033744026441394036 |
|
}, |
|
"harness|hendrycksTest-high_school_geography|5": { |
|
"acc": 0.7727272727272727, |
|
"acc_stderr": 0.029857515673386417, |
|
"acc_norm": 0.7727272727272727, |
|
"acc_norm_stderr": 0.029857515673386417 |
|
}, |
|
"harness|hendrycksTest-high_school_government_and_politics|5": { |
|
"acc": 0.8497409326424871, |
|
"acc_stderr": 0.025787723180723875, |
|
"acc_norm": 0.8497409326424871, |
|
"acc_norm_stderr": 0.025787723180723875 |
|
}, |
|
"harness|hendrycksTest-high_school_macroeconomics|5": { |
|
"acc": 0.6384615384615384, |
|
"acc_stderr": 0.024359581465396997, |
|
"acc_norm": 0.6384615384615384, |
|
"acc_norm_stderr": 0.024359581465396997 |
|
}, |
|
"harness|hendrycksTest-high_school_mathematics|5": { |
|
"acc": 0.337037037037037, |
|
"acc_stderr": 0.028820884666253255, |
|
"acc_norm": 0.337037037037037, |
|
"acc_norm_stderr": 0.028820884666253255 |
|
}, |
|
"harness|hendrycksTest-high_school_microeconomics|5": { |
|
"acc": 0.6764705882352942, |
|
"acc_stderr": 0.03038835355188679, |
|
"acc_norm": 0.6764705882352942, |
|
"acc_norm_stderr": 0.03038835355188679 |
|
}, |
|
"harness|hendrycksTest-high_school_physics|5": { |
|
"acc": 0.3443708609271523, |
|
"acc_stderr": 0.038796870240733264, |
|
"acc_norm": 0.3443708609271523, |
|
"acc_norm_stderr": 0.038796870240733264 |
|
}, |
|
"harness|hendrycksTest-high_school_psychology|5": { |
|
"acc": 0.8055045871559633, |
|
"acc_stderr": 0.01697028909045804, |
|
"acc_norm": 0.8055045871559633, |
|
"acc_norm_stderr": 0.01697028909045804 |
|
}, |
|
"harness|hendrycksTest-high_school_statistics|5": { |
|
"acc": 0.5370370370370371, |
|
"acc_stderr": 0.03400603625538272, |
|
"acc_norm": 0.5370370370370371, |
|
"acc_norm_stderr": 0.03400603625538272 |
|
}, |
|
"harness|hendrycksTest-high_school_us_history|5": { |
|
"acc": 0.7794117647058824, |
|
"acc_stderr": 0.02910225438967407, |
|
"acc_norm": 0.7794117647058824, |
|
"acc_norm_stderr": 0.02910225438967407 |
|
}, |
|
"harness|hendrycksTest-high_school_world_history|5": { |
|
"acc": 0.759493670886076, |
|
"acc_stderr": 0.027820781981149685, |
|
"acc_norm": 0.759493670886076, |
|
"acc_norm_stderr": 0.027820781981149685 |
|
}, |
|
"harness|hendrycksTest-human_aging|5": { |
|
"acc": 0.6636771300448431, |
|
"acc_stderr": 0.031708824268455, |
|
"acc_norm": 0.6636771300448431, |
|
"acc_norm_stderr": 0.031708824268455 |
|
}, |
|
"harness|hendrycksTest-human_sexuality|5": { |
|
"acc": 0.7251908396946565, |
|
"acc_stderr": 0.03915345408847836, |
|
"acc_norm": 0.7251908396946565, |
|
"acc_norm_stderr": 0.03915345408847836 |
|
}, |
|
"harness|hendrycksTest-international_law|5": { |
|
"acc": 0.71900826446281, |
|
"acc_stderr": 0.04103203830514512, |
|
"acc_norm": 0.71900826446281, |
|
"acc_norm_stderr": 0.04103203830514512 |
|
}, |
|
"harness|hendrycksTest-jurisprudence|5": { |
|
"acc": 0.7685185185185185, |
|
"acc_stderr": 0.04077494709252626, |
|
"acc_norm": 0.7685185185185185, |
|
"acc_norm_stderr": 0.04077494709252626 |
|
}, |
|
"harness|hendrycksTest-logical_fallacies|5": { |
|
"acc": 0.7177914110429447, |
|
"acc_stderr": 0.03536117886664743, |
|
"acc_norm": 0.7177914110429447, |
|
"acc_norm_stderr": 0.03536117886664743 |
|
}, |
|
"harness|hendrycksTest-machine_learning|5": { |
|
"acc": 0.41964285714285715, |
|
"acc_stderr": 0.04684099321077106, |
|
"acc_norm": 0.41964285714285715, |
|
"acc_norm_stderr": 0.04684099321077106 |
|
}, |
|
"harness|hendrycksTest-management|5": { |
|
"acc": 0.7766990291262136, |
|
"acc_stderr": 0.04123553189891431, |
|
"acc_norm": 0.7766990291262136, |
|
"acc_norm_stderr": 0.04123553189891431 |
|
}, |
|
"harness|hendrycksTest-marketing|5": { |
|
"acc": 0.8974358974358975, |
|
"acc_stderr": 0.019875655027867447, |
|
"acc_norm": 0.8974358974358975, |
|
"acc_norm_stderr": 0.019875655027867447 |
|
}, |
|
"harness|hendrycksTest-medical_genetics|5": { |
|
"acc": 0.71, |
|
"acc_stderr": 0.045604802157206845, |
|
"acc_norm": 0.71, |
|
"acc_norm_stderr": 0.045604802157206845 |
|
}, |
|
"harness|hendrycksTest-miscellaneous|5": { |
|
"acc": 0.7918263090676884, |
|
"acc_stderr": 0.014518592248904033, |
|
"acc_norm": 0.7918263090676884, |
|
"acc_norm_stderr": 0.014518592248904033 |
|
}, |
|
"harness|hendrycksTest-moral_disputes|5": { |
|
"acc": 0.7052023121387283, |
|
"acc_stderr": 0.024547617794803828, |
|
"acc_norm": 0.7052023121387283, |
|
"acc_norm_stderr": 0.024547617794803828 |
|
}, |
|
"harness|hendrycksTest-moral_scenarios|5": { |
|
"acc": 0.4044692737430168, |
|
"acc_stderr": 0.016414440917293147, |
|
"acc_norm": 0.4044692737430168, |
|
"acc_norm_stderr": 0.016414440917293147 |
|
}, |
|
"harness|hendrycksTest-nutrition|5": { |
|
"acc": 0.7091503267973857, |
|
"acc_stderr": 0.02600480036395213, |
|
"acc_norm": 0.7091503267973857, |
|
"acc_norm_stderr": 0.02600480036395213 |
|
}, |
|
"harness|hendrycksTest-philosophy|5": { |
|
"acc": 0.707395498392283, |
|
"acc_stderr": 0.02583989833487798, |
|
"acc_norm": 0.707395498392283, |
|
"acc_norm_stderr": 0.02583989833487798 |
|
}, |
|
"harness|hendrycksTest-prehistory|5": { |
|
"acc": 0.6944444444444444, |
|
"acc_stderr": 0.025630824975621355, |
|
"acc_norm": 0.6944444444444444, |
|
"acc_norm_stderr": 0.025630824975621355 |
|
}, |
|
"harness|hendrycksTest-professional_accounting|5": { |
|
"acc": 0.4716312056737589, |
|
"acc_stderr": 0.029779450957303055, |
|
"acc_norm": 0.4716312056737589, |
|
"acc_norm_stderr": 0.029779450957303055 |
|
}, |
|
"harness|hendrycksTest-professional_law|5": { |
|
"acc": 0.4302477183833116, |
|
"acc_stderr": 0.012645361435115233, |
|
"acc_norm": 0.4302477183833116, |
|
"acc_norm_stderr": 0.012645361435115233 |
|
}, |
|
"harness|hendrycksTest-professional_medicine|5": { |
|
"acc": 0.6397058823529411, |
|
"acc_stderr": 0.02916312857067073, |
|
"acc_norm": 0.6397058823529411, |
|
"acc_norm_stderr": 0.02916312857067073 |
|
}, |
|
"harness|hendrycksTest-professional_psychology|5": { |
|
"acc": 0.6470588235294118, |
|
"acc_stderr": 0.01933314202079716, |
|
"acc_norm": 0.6470588235294118, |
|
"acc_norm_stderr": 0.01933314202079716 |
|
}, |
|
"harness|hendrycksTest-public_relations|5": { |
|
"acc": 0.6363636363636364, |
|
"acc_stderr": 0.04607582090719976, |
|
"acc_norm": 0.6363636363636364, |
|
"acc_norm_stderr": 0.04607582090719976 |
|
}, |
|
"harness|hendrycksTest-security_studies|5": { |
|
"acc": 0.6775510204081633, |
|
"acc_stderr": 0.029923100563683906, |
|
"acc_norm": 0.6775510204081633, |
|
"acc_norm_stderr": 0.029923100563683906 |
|
}, |
|
"harness|hendrycksTest-sociology|5": { |
|
"acc": 0.8208955223880597, |
|
"acc_stderr": 0.027113286753111844, |
|
"acc_norm": 0.8208955223880597, |
|
"acc_norm_stderr": 0.027113286753111844 |
|
}, |
|
"harness|hendrycksTest-us_foreign_policy|5": { |
|
"acc": 0.85, |
|
"acc_stderr": 0.03588702812826371, |
|
"acc_norm": 0.85, |
|
"acc_norm_stderr": 0.03588702812826371 |
|
}, |
|
"harness|hendrycksTest-virology|5": { |
|
"acc": 0.5, |
|
"acc_stderr": 0.03892494720807614, |
|
"acc_norm": 0.5, |
|
"acc_norm_stderr": 0.03892494720807614 |
|
}, |
|
"harness|hendrycksTest-world_religions|5": { |
|
"acc": 0.8245614035087719, |
|
"acc_stderr": 0.029170885500727665, |
|
"acc_norm": 0.8245614035087719, |
|
"acc_norm_stderr": 0.029170885500727665 |
|
}, |
|
"harness|truthfulqa:mc|0": { |
|
"mc1": 0.3157894736842105, |
|
"mc1_stderr": 0.016272287957916923, |
|
"mc2": 0.4868797251828956, |
|
"mc2_stderr": 0.01529943410920313 |
|
}, |
|
"harness|winogrande|5": { |
|
"acc": 0.7813733228097869, |
|
"acc_stderr": 0.011616198215773239 |
|
}, |
|
"harness|gsm8k|5": { |
|
"acc": 0.36087945413191813, |
|
"acc_stderr": 0.013228626753925143 |
|
} |
|
} |
|
``` |
|
|
|
## 🧩 Configuration |
|
|
|
```yaml |
|
slices: |
|
- sources: |
|
- model: HuggingFaceH4/mistral-7b-anthropic |
|
layer_range: [0, 32] |
|
- model: HuggingFaceH4/mistral-7b-grok |
|
layer_range: [0, 32] |
|
merge_method: slerp |
|
base_model: HuggingFaceH4/mistral-7b-anthropic |
|
parameters: |
|
t: |
|
- filter: self_attn |
|
value: [0, 0.5, 0.3, 0.7, 1] |
|
- filter: mlp |
|
value: [1, 0.5, 0.7, 0.3, 0] |
|
- value: 0.5 |
|
dtype: bfloat16 |
|
``` |
|
|
|
## 💻 Usage |
|
|
|
```python |
|
!pip install -qU transformers accelerate |
|
|
|
from transformers import AutoTokenizer |
|
import transformers |
|
import torch |
|
|
|
model = "Isaak-Carter/JOSIE_Beta-8-7B-slerp" |
|
messages = [{"role": "user", "content": "What is a large language model?"}] |
|
|
|
tokenizer = AutoTokenizer.from_pretrained(model) |
|
prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) |
|
pipeline = transformers.pipeline( |
|
"text-generation", |
|
model=model, |
|
torch_dtype=torch.float16, |
|
device_map="auto", |
|
) |
|
|
|
outputs = pipeline(prompt, max_new_tokens=256, do_sample=True, temperature=0.7, top_k=50, top_p=0.95) |
|
print(outputs[0]["generated_text"]) |
|
``` |
