Update README.md
Browse filesadding a comment that cuda kernels needs to be build
README.md
CHANGED
@@ -24,6 +24,9 @@ tokenizer = AutoTokenizer.from_pretrained(model_id)
|
|
24 |
model = HQQModelForCausalLM.from_quantized(model_id)
|
25 |
|
26 |
#Optional: set backend/compile
|
|
|
|
|
|
|
27 |
from hqq.core.quantize import *
|
28 |
HQQLinear.set_backend(HQQBackend.ATEN_BACKPROP)
|
29 |
|
|
|
24 |
model = HQQModelForCausalLM.from_quantized(model_id)
|
25 |
|
26 |
#Optional: set backend/compile
|
27 |
+
#You will need to install CUDA kernels apriori
|
28 |
+
# git clone https://github.com/mobiusml/hqq/
|
29 |
+
# cd hqq/kernels && python setup_cuda.py install
|
30 |
from hqq.core.quantize import *
|
31 |
HQQLinear.set_backend(HQQBackend.ATEN_BACKPROP)
|
32 |
|