abhinavkulkarni
commited on
Commit
•
8688d11
1
Parent(s):
e32470f
Update README.md
Browse files
README.md
CHANGED
@@ -40,7 +40,7 @@ import torch
|
|
40 |
from awq.quantize.quantizer import real_quantize_model_weight
|
41 |
from transformers import AutoModelForCausalLM, AutoConfig, AutoTokenizer
|
42 |
from accelerate import init_empty_weights, load_checkpoint_and_dispatch
|
43 |
-
from huggingface_hub import
|
44 |
|
45 |
model_name = "tiiuae/falcon-40b-instruct"
|
46 |
|
@@ -57,7 +57,7 @@ q_config = {
|
|
57 |
"q_group_size": 64,
|
58 |
}
|
59 |
|
60 |
-
load_quant =
|
61 |
|
62 |
with init_empty_weights():
|
63 |
model = AutoModelForCausalLM.from_config(config=config,
|
|
|
40 |
from awq.quantize.quantizer import real_quantize_model_weight
|
41 |
from transformers import AutoModelForCausalLM, AutoConfig, AutoTokenizer
|
42 |
from accelerate import init_empty_weights, load_checkpoint_and_dispatch
|
43 |
+
from huggingface_hub import snapshot_download
|
44 |
|
45 |
model_name = "tiiuae/falcon-40b-instruct"
|
46 |
|
|
|
57 |
"q_group_size": 64,
|
58 |
}
|
59 |
|
60 |
+
load_quant = snapshot_download('abhinavkulkarni/tiiuae-falcon-40b-instruct-w4-g128-awq', 'pytorch_model.bin')
|
61 |
|
62 |
with init_empty_weights():
|
63 |
model = AutoModelForCausalLM.from_config(config=config,
|