| { | |
| "current_gpu_type": "NVIDIA L40S", | |
| "current_gpu_total_memory": 45372.6875, | |
| "memory_inference_first": 25590.0, | |
| "memory_inference": 25590.0, | |
| "token_generation_latency_sync": 30.128338623046876, | |
| "token_generation_latency_async": 30.288023874163628, | |
| "token_generation_throughput_sync": 0.033191342294428515, | |
| "token_generation_throughput_async": 0.03301635009780294, | |
| "token_generation_CO2_emissions": 3.236854331824288e-06, | |
| "token_generation_energy_consumption": 0.001333430427257508, | |
| "inference_latency_sync": 23.984115219116212, | |
| "inference_latency_async": 22.998762130737305, | |
| "inference_throughput_sync": 0.04169426267611338, | |
| "inference_throughput_async": 0.0434806010130225, | |
| "inference_CO2_emissions": 2.2244397934888317e-06, | |
| "inference_energy_consumption": 7.886087285242966e-06 | |
| } |