Abhinav Kulkarni
commited on
Commit
•
69f4c6e
1
Parent(s):
137bf22
Updated README
Browse files
README.md
CHANGED
@@ -27,9 +27,9 @@ This model was successfully tested on CUDA driver v530.30.02 and runtime v11.7 w
|
|
27 |
## How to Use
|
28 |
|
29 |
```bash
|
30 |
-
git clone https://github.com/
|
31 |
&& cd llm-awq \
|
32 |
-
&& git checkout
|
33 |
&& pip install -e . \
|
34 |
&& cd awq/kernels \
|
35 |
&& python setup.py install
|
@@ -40,9 +40,9 @@ import torch
|
|
40 |
from awq.quantize.quantizer import real_quantize_model_weight
|
41 |
from transformers import AutoModelForCausalLM, AutoConfig, AutoTokenizer
|
42 |
from accelerate import init_empty_weights, load_checkpoint_and_dispatch
|
43 |
-
from huggingface_hub import
|
44 |
|
45 |
-
model_name = "tiiuae
|
46 |
|
47 |
# Config
|
48 |
config = AutoConfig.from_pretrained(model_name, trust_remote_code=True)
|
@@ -57,7 +57,7 @@ q_config = {
|
|
57 |
"q_group_size": 64,
|
58 |
}
|
59 |
|
60 |
-
load_quant =
|
61 |
|
62 |
with init_empty_weights():
|
63 |
model = AutoModelForCausalLM.from_config(config=config,
|
|
|
27 |
## How to Use
|
28 |
|
29 |
```bash
|
30 |
+
git clone https://github.com/abhinavkulkarni/llm-awq \
|
31 |
&& cd llm-awq \
|
32 |
+
&& git checkout e977c5a570c5048b67a45b1eb823b81de02d0d60 \
|
33 |
&& pip install -e . \
|
34 |
&& cd awq/kernels \
|
35 |
&& python setup.py install
|
|
|
40 |
from awq.quantize.quantizer import real_quantize_model_weight
|
41 |
from transformers import AutoModelForCausalLM, AutoConfig, AutoTokenizer
|
42 |
from accelerate import init_empty_weights, load_checkpoint_and_dispatch
|
43 |
+
from huggingface_hub import snapshot_download
|
44 |
|
45 |
+
model_name = "abhinavkulkarni/tiiuae-falcon-7b-instruct-w4-g64-awq"
|
46 |
|
47 |
# Config
|
48 |
config = AutoConfig.from_pretrained(model_name, trust_remote_code=True)
|
|
|
57 |
"q_group_size": 64,
|
58 |
}
|
59 |
|
60 |
+
load_quant = snapshot_download(model_name)
|
61 |
|
62 |
with init_empty_weights():
|
63 |
model = AutoModelForCausalLM.from_config(config=config,
|