File size: 277 Bytes
539ee6f 18b0674 be03f7f 539ee6f |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
from llama_cpp import Llama
import os
model_path = "llama-3.2-1B-it-Ecommerce-ChatBot-merged-F16.gguf"
n_threads = os.cpu_count()
llm = Llama(
model_path=model_path,
n_ctx=512,
n_batch=64,
n_threads=n_threads,
n_gpu_layers=-1,
chat_format="llama-3"
)
|