Added accelerate

app.py

@@ -1,24 +1,15 @@
 import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
+from transformers import AutoModelForCausalLM, AutoTokenizer
 from peft import PeftConfig, PeftModel
 import gradio as gr
 
 # Loading PEFT model
 PEFT_MODEL = "gunjanjoshi/llama2-7b-sharded-bf16-finetuned-mental-health-conversational"
 
-# Modify BitsAndBytesConfig for CPU
-bnb_config = BitsAndBytesConfig(
-    load_in_4bit=True,
-    bnb_4bit_quant_type="pf4",
-    bnb_4bit_use_double_quant=True,
-    bnb_4bit_compute_dtype=torch.float16,
-)
-
 config = PeftConfig.from_pretrained(PEFT_MODEL)
 peft_base_model = AutoModelForCausalLM.from_pretrained(
     config.base_model_name_or_path,
     return_dict=True,
-    quantization_config=bnb_config,
     device_map="cpu",  # Ensure this is set to CPU
     trust_remote_code=True,
 )