| { | |
| "current_gpu_type": "NVIDIA L40S", | |
| "current_gpu_total_memory": 45372.6875, | |
| "memory_inference_first": 4312.0, | |
| "memory_inference": 4208.0, | |
| "token_generation_latency_sync": 47.82877426147461, | |
| "token_generation_latency_async": 48.09337295591831, | |
| "token_generation_throughput_sync": 0.020907916112027268, | |
| "token_generation_throughput_async": 0.02079288555861086, | |
| "token_generation_CO2_emissions": 3.164995601682604e-06, | |
| "token_generation_energy_consumption": 0.00230724002830658, | |
| "inference_latency_sync": 41.39498291015625, | |
| "inference_latency_async": 40.357160568237305, | |
| "inference_throughput_sync": 0.02415751691866625, | |
| "inference_throughput_async": 0.024778750187569934, | |
| "inference_CO2_emissions": 3.351062981628799e-06, | |
| "inference_energy_consumption": 1.1458303099082523e-05 | |
| } |