abhinavkulkarni
/

tiiuae-falcon-7b-instruct-w4-g64-awq

Text Generation

RefinedWebModel

text-generation-inference

Model card Files Files and versions Community

abhinavkulkarni commited on Jul 9, 2023

Commit

8f8f68c

•

1 Parent(s): c0eee97

Update README.md

Files changed (1) hide show

README.md +1 -1

README.md CHANGED Viewed

@@ -63,7 +63,7 @@ q_config = {
 load_quant = hf_hub_download('abhinavkulkarni/falcon-7b-instruct-w4-g64-awq', 'pytorch_model.bin')
 with init_empty_weights():
-    model = AutoModelForCausalLM.from_pretrained(model_name, config=config,
                                                  torch_dtype=torch.float16, trust_remote_code=True)
 real_quantize_model_weight(model, w_bit=w_bit, q_config=q_config, init_only=True)

 load_quant = hf_hub_download('abhinavkulkarni/falcon-7b-instruct-w4-g64-awq', 'pytorch_model.bin')
 with init_empty_weights():
+    model = AutoModelForCausalLM.from_config(config=config,
                                                  torch_dtype=torch.float16, trust_remote_code=True)
 real_quantize_model_weight(model, w_bit=w_bit, q_config=q_config, init_only=True)