Spaces:
Runtime error
Runtime error
gunjanjoshi
commited on
Commit
·
0973b84
1
Parent(s):
347544f
Added accelerate
Browse files
app.py
CHANGED
@@ -9,7 +9,7 @@ PEFT_MODEL = "gunjanjoshi/llama2-7b-sharded-bf16-finetuned-mental-health-convers
|
|
9 |
# Modify BitsAndBytesConfig for CPU
|
10 |
bnb_config = BitsAndBytesConfig(
|
11 |
load_in_4bit=True,
|
12 |
-
bnb_4bit_quant_type="
|
13 |
bnb_4bit_use_double_quant=True,
|
14 |
bnb_4bit_compute_dtype=torch.float16,
|
15 |
)
|
|
|
9 |
# Modify BitsAndBytesConfig for CPU
|
10 |
bnb_config = BitsAndBytesConfig(
|
11 |
load_in_4bit=True,
|
12 |
+
bnb_4bit_quant_type="pf4",
|
13 |
bnb_4bit_use_double_quant=True,
|
14 |
bnb_4bit_compute_dtype=torch.float16,
|
15 |
)
|