File size: 1,774 Bytes
e029e22 f51bb92 b83cc65 6a3dbe6 6158da4 b83cc65 6158da4 6a3dbe6 d1afae8 1afc595 6a3dbe6 6158da4 9b7a7cf e029e22 b83cc65 1afc595 b83cc65 1e2550f b83cc65 6158da4 e029e22 6158da4 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 |
from langchain_openai import ChatOpenAI
from langchain_community.llms import LlamaCpp
import os
from pathlib import Path
from huggingface_hub import hf_hub_download
class ChatModelLoader:
def __init__(self, config):
self.config = config
self.huggingface_token = os.getenv("HUGGINGFACEHUB_API_TOKEN")
def _verify_model_cache(self, model_cache_path):
hf_hub_download(
repo_id=self.config["llm_params"]["local_llm_params"]["repo_id"],
filename=self.config["llm_params"]["local_llm_params"]["filename"],
cache_dir=model_cache_path,
)
return str(list(Path(model_cache_path).glob("*/snapshots/*/*.gguf"))[0])
def load_chat_model(self):
if self.config["llm_params"]["llm_loader"] in [
"gpt-3.5-turbo-1106",
"gpt-4",
"gpt-4o-mini",
]:
llm = ChatOpenAI(model_name=self.config["llm_params"]["llm_loader"])
elif self.config["llm_params"]["llm_loader"] == "local_llm":
n_batch = 512 # Should be between 1 and n_ctx, consider the amount of VRAM in your GPU.
model_path = self._verify_model_cache(
self.config["llm_params"]["local_llm_params"]["model"]
)
llm = LlamaCpp(
model_path=model_path,
n_batch=n_batch,
n_ctx=2048,
f16_kv=True,
verbose=True,
n_threads=2,
temperature=self.config["llm_params"]["local_llm_params"][
"temperature"
],
)
else:
raise ValueError(
f"Invalid LLM Loader: {self.config['llm_params']['llm_loader']}"
)
return llm
|