config_file.json

{
    "host": "0.0.0.0",
    "port": 7680,
    "models": [
        {
            "model": "/home/user/app/DeepSeek-R1-Distill-Llama-8B-Q4_K_L.gguf",
            "model_alias": "DeepSeek-R1-Distill-Llama-8B-Q4_K_L",
            "chat_format": "chatml",
            "n_gpu_layers": -1,
            "offload_kqv": true,
            "n_threads": 12,
            "n_batch": 512,
            "n_ctx": 512
        }
    ]
}