|
--- |
|
tags: |
|
- deepsparse |
|
--- |
|
## Usage |
|
|
|
```python |
|
from deepsparse import TextGeneration |
|
|
|
prompt = "How to make banana bread?" |
|
formatted_prompt = f"<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant\n" |
|
|
|
model = TextGeneration(model="hf:nm-testing/TinyLlama-1.1B-Chat-v0.4-pruned50-quant") |
|
print(model(formatted_prompt, max_new_tokens=300).generations[0].text) |
|
|
|
""" |
|
Banana bread is a delicious and healthy recipe that is easy to make. Here is a recipe for banana bread: |
|
|
|
Ingredients: |
|
|
|
1. 15 bananas |
|
2. 1 cup of mashed bananas |
|
3. 1 cup of milk |
|
4. 1 cup of eggs |
|
5. 1 cup of sugar |
|
6. 1 cup of flour |
|
7. 1 teaspoon of baking powder |
|
8. 1 teaspoon of baking sugar |
|
9. 1 cup of whole wheat flour |
|
10. 1 cup of whole wheat sugar |
|
11. 1 cup of whole wheat flour |
|
12. 1 cup of whole wheat flour |
|
13. 1 cup of whole wheat flour |
|
14. 1 cup of whole wheat flour |
|
15. 1 cup of whole wheat flour |
|
|
|
Step 1: Preheat the oven to 325°F (164°C). |
|
|
|
Step 2: Grease a 10- to 12-inch (25- to 35-centimeter) round baking pan. |
|
|
|
Step 3: Add the ingredients for the batter. |
|
|
|
Step 4: Pour the batter into the prepared pan. |
|
|
|
Step 5 |
|
""" |
|
``` |
|
|
|
## One-shot and Export |
|
|
|
``` |
|
git clone https://github.com/neuralmagic/sparseml |
|
pip install -e "sparseml[transformers]" "torch<2" |
|
cd sparseml |
|
git checkout update/onnx_export/duplicate |
|
python src/sparseml/transformers/sparsification/obcq/obcq.py TinyLlama/TinyLlama-1.1B-Chat-v0.4 open_platypus --recipe recipe.yaml --save True |
|
python src/sparseml/transformers/sparsification/obcq/export.py --task text-generation --model_path obcq_deployment |
|
cp deployment/model.onnx deployment/model-orig.onnx |
|
python onnx_kv_inject.py --input-file deployment/model-orig.onnx --output-file deployment/model.onnx |
|
``` |
