Abhinav Kulkarni
commited on
Commit
•
6bfceae
1
Parent(s):
fcdb405
Updated README
Browse files
README.md
CHANGED
|
@@ -30,9 +30,9 @@ For Docker users, the `nvcr.io/nvidia/pytorch:23.06-py3` image is runtime v12.1
|
|
| 30 |
## How to Use
|
| 31 |
|
| 32 |
```bash
|
| 33 |
-
git clone https://github.com/
|
| 34 |
&& cd llm-awq \
|
| 35 |
-
&& git checkout
|
| 36 |
&& pip install -e . \
|
| 37 |
&& cd awq/kernels \
|
| 38 |
&& python setup.py install
|
|
@@ -45,7 +45,7 @@ from transformers import AutoModelForCausalLM, AutoConfig, AutoTokenizer
|
|
| 45 |
from accelerate import init_empty_weights, load_checkpoint_and_dispatch
|
| 46 |
from huggingface_hub import snapshot_download
|
| 47 |
|
| 48 |
-
model_name = "mosaicml
|
| 49 |
|
| 50 |
# Config
|
| 51 |
config = AutoConfig.from_pretrained(model_name, trust_remote_code=True)
|
|
@@ -60,7 +60,7 @@ q_config = {
|
|
| 60 |
"q_group_size": 128,
|
| 61 |
}
|
| 62 |
|
| 63 |
-
load_quant = snapshot_download(
|
| 64 |
|
| 65 |
with init_empty_weights():
|
| 66 |
model = AutoModelForCausalLM.from_config(config=config,
|
|
|
|
| 30 |
## How to Use
|
| 31 |
|
| 32 |
```bash
|
| 33 |
+
git clone https://github.com/abhinavkulkarni/llm-awq \
|
| 34 |
&& cd llm-awq \
|
| 35 |
+
&& git checkout e977c5a570c5048b67a45b1eb823b81de02d0d60 \
|
| 36 |
&& pip install -e . \
|
| 37 |
&& cd awq/kernels \
|
| 38 |
&& python setup.py install
|
|
|
|
| 45 |
from accelerate import init_empty_weights, load_checkpoint_and_dispatch
|
| 46 |
from huggingface_hub import snapshot_download
|
| 47 |
|
| 48 |
+
model_name = "abhinavkulkarni/mosaicml-mpt-30b-chat-w4-g128-awq"
|
| 49 |
|
| 50 |
# Config
|
| 51 |
config = AutoConfig.from_pretrained(model_name, trust_remote_code=True)
|
|
|
|
| 60 |
"q_group_size": 128,
|
| 61 |
}
|
| 62 |
|
| 63 |
+
load_quant = snapshot_download(model_name)
|
| 64 |
|
| 65 |
with init_empty_weights():
|
| 66 |
model = AutoModelForCausalLM.from_config(config=config,
|