File size: 663 Bytes
07ea411 28d1b45 07ea411 9f300f4 07ea411 9f300f4 07ea411 0b6a9e8 28d1b45 9f300f4 28d1b45 07ea411 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 |
{
"parameters": {
"max_total_tokens": 4096, // Increase from 2048
"max_input_length": 2048, // Increase from 1024
"max_batch_total_tokens": 16384, // Increase from 8192
"max_concurrent_requests": 2, // Increase from 1
"max_batch_size": 2, // Increase from 1
"waiting_served_ratio": 0.8 // Decrease from 1.2
},
"hardware": {
"task_type": "text-generation",
"accelerator": "gpu",
"num_gpus": 1,
"gpu_memory_gb": 24,
"distributed_setup": false
},
"framework_type": "pytorch",
"torch_compile": true,
"trust_remote_code": true,
"disable_custom_kernels": false
} |