framework,version,device,op_name,kernel_source,batch_size,isl,num_heads,num_key_value_heads,head_dim,beam_width,attn_dtype,kv_cache_dtype,step,latency
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,1,float16,float16,1,0.03311466674009959
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,1,float16,float16,7,0.03310399999221166
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,1,float16,fp8,7,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,1,float16,float16,3,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,1,float16,fp8,1,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,1,float16,fp8,3,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,1,float16,float16,15,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,1,float16,float16,31,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,1,float16,fp8,15,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,1,float16,fp8,31,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,1,float16,float16,63,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,1,float16,fp8,63,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,1,float16,float16,127,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,1,float16,fp8,127,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,1,float16,float16,255,0.03549866626660029
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,1,float16,fp8,255,0.02696000039577484
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,1,float16,float16,511,0.053930665055910744
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,1,float16,fp8,511,0.03994133323431015
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,1,float16,float16,1023,0.09181867043177287
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,1,float16,fp8,1023,0.05836800237496694
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,1,float16,float16,2047,0.17373865842819214
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,1,float16,fp8,2047,0.10069333513577779
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,1,float16,float16,4095,0.3285333315531413
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,1,float16,float16,1,0.013728000223636627
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,1,float16,fp8,4095,0.17425066232681274
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,1,float16,float16,3,0.010821333775917688
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,1,float16,fp8,1,0.010858666151762009
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,1,float16,fp8,3,0.012698666503032049
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,1,float16,float16,7,0.010858666151762009
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,1,float16,fp8,7,0.010922666639089584
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,1,float16,float16,15,0.010863999525705973
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,1,float16,fp8,15,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,1,float16,float16,31,0.010512000570694605
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,1,float16,fp8,31,0.012533333152532578
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,1,float16,float16,63,0.011605333536863327
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,1,float16,fp8,63,0.013141332815090815
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,1,float16,float16,127,0.010522666076819101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,1,float16,fp8,127,0.01098666712641716
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,1,float16,float16,255,0.010853332777818045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,1,float16,fp8,255,0.012351999680201212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,1,float16,fp8,511,0.01302933320403099
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,1,float16,float16,511,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,1,float16,float16,1023,0.012805332740147909
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,1,float16,fp8,1023,0.012869333227475485
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,1,float16,fp8,2047,0.014842666685581207
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,1,float16,float16,4095,0.019050666441520054
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,1,float16,fp8,4095,0.018863999595244724
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,1,float16,float16,8191,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,1,float16,fp8,8191,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,1,float16,fp8,16383,0.03515200068553289
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,1,float16,float16,16383,0.05578133463859558
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,1,float16,float16,1,0.012730666746695837
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,1,float16,fp8,1,0.010863999525705973
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,1,float16,float16,3,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,1,float16,fp8,3,0.010853332777818045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,1,float16,float16,7,0.010687999427318573
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,1,float16,fp8,7,0.012533333152532578
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,1,float16,fp8,15,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,1,float16,float16,31,0.012357333054145178
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,1,float16,float16,15,0.010863999525705973
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,1,float16,fp8,31,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,1,float16,float16,63,0.012341332932313284
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,1,float16,fp8,63,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,1,float16,float16,127,0.010933333386977514
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,1,float16,fp8,127,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,1,float16,float16,255,0.012741333494583765
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,1,float16,fp8,255,0.01097600037852923
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,1,float16,float16,511,0.012863999853531519
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,1,float16,fp8,511,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,1,float16,float16,1023,0.014736000448465347
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,1,float16,fp8,1023,0.014570667097965876
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,1,float16,float16,2047,0.019061333189407986
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,1,float16,fp8,2047,0.01870399961868922
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,1,float16,float16,4095,0.03549866626660029
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,1,float16,fp8,4095,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,1,float16,fp8,8191,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,1,float16,float16,8191,0.05734399954477946
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,1,float16,float16,16383,0.0993280013402303
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,1,float16,fp8,16383,0.05972800155480703
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,1,float16,float16,1,0.05563733478387197
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,1,float16,fp8,1,0.04131199916203817
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,1,float16,fp8,3,0.043354665239652
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,1,float16,float16,7,0.05563733478387197
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,1,float16,fp8,7,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,1,float16,float16,15,0.055642664432525635
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,1,float16,fp8,15,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,1,float16,float16,31,0.05563733478387197
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,1,float16,fp8,31,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,1,float16,float16,63,0.05563200016816457
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,1,float16,fp8,63,0.043354665239652
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,1,float16,float16,127,0.0576853354771932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,1,float16,fp8,127,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,1,float16,float16,255,0.0576853354771932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,1,float16,fp8,255,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,1,float16,fp8,511,0.06587733328342438
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,1,float16,float16,511,0.09284266829490662
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,1,float16,float16,1023,0.16776533921559653
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,1,float16,fp8,1023,0.09967999656995137
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,1,float16,float16,2047,0.32614932457606
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,1,float16,fp8,2047,0.1771519978841146
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,1,float16,float16,1,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,1,float16,fp8,1,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,1,float16,float16,3,0.01462399959564209
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,1,float16,fp8,3,0.012746666868527731
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,1,float16,float16,7,0.01461333284775416
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,1,float16,fp8,7,0.012709333250919977
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,1,float16,float16,15,0.01451733335852623
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,1,float16,fp8,15,0.012709333250919977
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,1,float16,float16,31,0.014501333236694336
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,1,float16,fp8,31,0.012928000340859095
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,1,float16,float16,63,0.0129120002190272
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,1,float16,fp8,63,0.012714666624863943
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,1,float16,float16,127,0.012906666845083237
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,1,float16,fp8,127,0.012810666114091873
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,1,float16,float16,255,0.01440000037352244
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,1,float16,fp8,255,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,1,float16,float16,511,0.016666666915019352
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,1,float16,fp8,511,0.0145066666106383
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,1,float16,float16,1023,0.02081599955757459
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,1,float16,fp8,1023,0.018021332720915478
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,1,float16,float16,2047,0.039594667653242745
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,1,float16,fp8,2047,0.024911999702453613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,1,float16,float16,4095,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,1,float16,fp8,4095,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,1,float16,float16,8191,0.10274133086204529
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,1,float16,fp8,8191,0.06177600224812826
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,1,float16,float16,16383,0.1868799924850464
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,1,float16,fp8,16383,0.10478933652242024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,1,float16,float16,1,0.10069333513577779
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,1,float16,float16,3,0.10069333513577779
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,1,float16,fp8,1,0.07407466570536296
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,1,float16,fp8,3,0.07406933108965556
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,1,float16,float16,7,0.10180266698201497
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,1,float16,fp8,7,0.07406933108965556
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,1,float16,float16,15,0.10171733299891154
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,1,float16,fp8,15,0.07406933108965556
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,1,float16,float16,31,0.10103999574979146
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,1,float16,fp8,31,0.07406933108965556
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,1,float16,float16,63,0.10103467106819153
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,1,float16,fp8,63,0.07406933108965556
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,1,float16,float16,127,0.10274133086204529
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,1,float16,fp8,127,0.07611733178297679
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,1,float16,float16,255,0.10274133086204529
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,1,float16,fp8,255,0.07611733178297679
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,1,float16,float16,511,0.17015467087427774
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,1,float16,fp8,511,0.11707733074824016
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,1,float16,float16,1023,0.3199999928474426
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,1,float16,fp8,1023,0.18705066045125326
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,1,float16,float16,1,0.1909760038057963
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,1,float16,fp8,1,0.13943466544151306
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,1,float16,float16,3,0.1909760038057963
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,1,float16,fp8,3,0.13943466544151306
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,1,float16,float16,7,0.1909760038057963
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,1,float16,fp8,7,0.1397760013739268
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,1,float16,float16,15,0.19165867567062378
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,1,float16,fp8,15,0.13942933082580566
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,1,float16,float16,31,0.19063466787338257
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,1,float16,fp8,31,0.1397760013739268
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,1,float16,float16,63,0.19063466787338257
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,1,float16,fp8,63,0.13960533340771994
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,1,float16,float16,127,0.19062934319178262
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,1,float16,fp8,127,0.13943466544151306
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,1,float16,float16,255,0.1930239995320638
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,1,float16,fp8,255,0.13943466544151306
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,1,float16,fp8,1,0.26641066869099933
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,1,float16,float16,1,0.36983466148376465
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,1,float16,float16,3,0.3667626778284709
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,1,float16,fp8,3,0.2664213379224141
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,1,float16,float16,7,0.3667626778284709
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,1,float16,fp8,7,0.2667520046234131
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,1,float16,float16,15,0.3667626778284709
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,1,float16,fp8,15,0.266757329305013
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,1,float16,float16,31,0.3667626778284709
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,1,float16,fp8,31,0.26641066869099933
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,1,float16,float16,63,0.3667626778284709
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,1,float16,fp8,63,0.2677813371022542
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,1,float16,float16,127,0.3667626778284709
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,1,float16,fp8,127,0.26470400889714557
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,1,float16,float16,1,0.016656000167131424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,1,float16,fp8,1,0.014869333555301031
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,1,float16,float16,3,0.016656000167131424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,1,float16,fp8,3,0.014746667196353277
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,1,float16,float16,7,0.016800000021855038
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,1,float16,fp8,7,0.014757333944241205
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,1,float16,float16,15,0.016901332885026932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,1,float16,fp8,15,0.014677333335081736
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,1,float16,float16,31,0.016719999412695568
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,1,float16,fp8,31,0.014746667196353277
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,1,float16,float16,63,0.016666666915019352
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,1,float16,fp8,63,0.014773332824309668
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,1,float16,float16,127,0.01666133354107539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,1,float16,fp8,127,0.014848000059525171
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,1,float16,float16,255,0.016901332885026932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,1,float16,fp8,255,0.014853333433469137
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,1,float16,float16,511,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,1,float16,fp8,511,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,1,float16,float16,1023,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,1,float16,fp8,1023,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,1,float16,float16,2047,0.06108266611893972
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,1,float16,fp8,2047,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,1,float16,float16,4095,0.10274133086204529
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,1,float16,fp8,4095,0.062122667829195656
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,1,float16,float16,8191,0.18824533621470133
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,1,float16,fp8,8191,0.10478400190671285
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,1,float16,float16,16383,0.3575413227081299
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,1,float16,fp8,16383,0.1904639999071757
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,1,float16,float16,1,0.7208960056304932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,1,float16,fp8,1,0.5190026760101318
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,1,float16,float16,3,0.719871997833252
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,1,float16,fp8,3,0.5193386475245158
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,1,float16,float16,7,0.7205599943796793
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,1,float16,fp8,7,0.520362655321757
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,1,float16,float16,15,0.7208906809488932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,1,float16,fp8,15,0.5193493366241455
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,1,float16,float16,31,0.7202133337656657
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,1,float16,fp8,31,0.5200213193893433
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,1,float16,float16,63,0.7215786774953207
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,1,float16,fp8,63,0.5207039912541708
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,128,1,float16,float16,1,1.427120049794515
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,128,1,float16,fp8,1,1.021440029144287
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,128,1,float16,float16,3,1.4264426231384277
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,128,1,float16,fp8,3,1.0224586327870686
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,128,1,float16,float16,7,1.4267733891805012
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,128,1,float16,fp8,7,1.0207573572794597
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,128,1,float16,float16,15,1.4271146456400554
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,128,1,float16,fp8,15,1.0217920144399006
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,128,1,float16,float16,31,1.4271094004313152
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,128,1,float16,fp8,31,1.0221227010091145
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,1,float16,float16,1,0.021503999829292297
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,1,float16,fp8,1,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,1,float16,float16,3,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,1,float16,fp8,3,0.018768000106016796
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,1,float16,float16,7,0.022863999009132385
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,1,float16,fp8,7,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,1,float16,float16,15,0.021146667500336964
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,1,float16,fp8,15,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,1,float16,float16,31,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,1,float16,fp8,31,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,1,float16,float16,63,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,1,float16,fp8,63,0.018709332992633183
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,1,float16,float16,127,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,1,float16,fp8,127,0.018778666853904724
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,1,float16,float16,255,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,1,float16,fp8,255,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,1,float16,float16,511,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,1,float16,fp8,511,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,1,float16,float16,1023,0.05358933409055074
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,1,float16,fp8,1023,0.035162667433420815
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,1,float16,float16,2047,0.0962559978167216
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,1,float16,fp8,2047,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,1,float16,float16,4095,0.17390932639439902
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,1,float16,fp8,4095,0.09660266836484273
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,1,float16,float16,8191,0.3285333315531413
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,1,float16,fp8,8191,0.17202132940292358
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,1,float16,float16,1,0.021503999829292297
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,1,float16,fp8,1,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,1,float16,float16,3,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,1,float16,fp8,3,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,1,float16,float16,7,0.022874665757020313
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,1,float16,fp8,7,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,1,float16,float16,15,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,1,float16,fp8,15,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,1,float16,float16,31,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,1,float16,fp8,31,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,1,float16,float16,63,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,1,float16,fp8,63,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,1,float16,float16,127,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,1,float16,fp8,127,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,1,float16,float16,255,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,1,float16,fp8,255,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,1,float16,float16,511,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,1,float16,fp8,511,0.024911999702453613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,1,float16,float16,1023,0.05358933409055074
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,1,float16,fp8,1023,0.03721066564321518
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,1,float16,float16,2047,0.01589866727590561
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,1,float16,float16,2047,0.09659733374913533
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,1,float16,fp8,2047,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,1,float16,float16,4095,0.1735573410987854
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,1,float16,fp8,4095,0.0976106623808543
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,1,float16,float16,8191,0.3278506596883138
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,1,float16,fp8,8191,0.1723733345667521
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,1,float16,float16,1,0.010741333166758219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,1,float16,fp8,1,0.010682666053374609
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,1,float16,float16,3,0.010816000401973724
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,1,float16,fp8,3,0.012159999459981918
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,1,float16,fp8,7,0.010645333677530289
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,1,float16,float16,7,0.010512000570694605
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,1,float16,float16,15,0.011749333391586939
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,1,float16,fp8,15,0.010645333677530289
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,1,float16,float16,31,0.010687999427318573
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,1,float16,fp8,31,0.0122079998254776
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,1,float16,float16,63,0.01101333275437355
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,1,float16,fp8,63,0.010687999427318573
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,1,float16,float16,127,0.01201066623131434
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,1,float16,fp8,127,0.012367999802033106
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,1,float16,float16,255,0.010677333921194077
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,1,float16,fp8,255,0.010693332801262537
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,1,float16,float16,511,0.012890666723251343
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,1,float16,fp8,511,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,1,float16,fp8,1023,0.01259200026591619
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,1,float16,float16,1023,0.012373333175977072
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,1,float16,float16,2047,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,1,float16,fp8,2047,0.014576000471909841
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,1,float16,float16,4095,0.016666666915019352
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,1,float16,fp8,4095,0.01461333284775416
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,1,float16,float16,8191,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,1,float16,fp8,8191,0.016666666915019352
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,1,float16,float16,16383,0.03516799956560135
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,1,float16,fp8,16383,0.022874665757020313
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,1,float16,float16,32767,0.055973331133524575
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,1,float16,fp8,32767,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,1,float16,float16,1,0.012576000144084295
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,1,float16,fp8,1,0.012847999731699625
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,1,float16,float16,3,0.05563733478387197
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,1,float16,fp8,3,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,1,float16,float16,3,0.010687999427318573
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,1,float16,float16,7,0.011765333513418833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,1,float16,fp8,7,0.012906666845083237
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,1,float16,float16,15,0.010981333752473196
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,1,float16,fp8,15,0.010853332777818045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,1,float16,float16,31,0.012138667205969492
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,1,float16,fp8,31,0.012213333199421564
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,1,float16,float16,63,0.010992000500361124
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,1,float16,fp8,63,0.010682666053374609
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,1,float16,float16,127,0.011674666156371435
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,1,float16,fp8,127,0.012272000312805176
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,1,float16,float16,255,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,1,float16,fp8,255,0.010672000547250112
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,1,float16,float16,511,0.012549333274364471
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,1,float16,fp8,511,0.013418667018413544
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,1,float16,float16,1023,0.012863999853531519
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,1,float16,fp8,1023,0.012794667234023413
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,1,float16,float16,2047,0.01672533278663953
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,1,float16,fp8,2047,0.016634666671355564
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,1,float16,float16,4095,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,1,float16,fp8,4095,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,1,float16,float16,8191,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,1,float16,fp8,8191,0.02492266645034154
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,1,float16,float16,16383,0.05769066512584686
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,1,float16,fp8,16383,0.03721066564321518
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,1,float16,float16,32767,0.09864532947540283
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,1,float16,float16,1,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,1,float16,fp8,32767,0.057349334160486855
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,1,float16,fp8,1,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,1,float16,float16,3,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,1,float16,fp8,3,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,1,float16,float16,7,0.03345066557327906
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,1,float16,fp8,7,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,1,float16,float16,15,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,1,float16,fp8,15,0.02697066714366277
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,1,float16,float16,31,0.03311466674009959
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,1,float16,fp8,31,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,1,float16,fp8,63,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,1,float16,float16,127,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,1,float16,fp8,127,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,1,float16,float16,255,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,1,float16,fp8,255,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,1,float16,fp8,511,0.039247999588648476
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,1,float16,float16,511,0.053930665055910744
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,1,float16,float16,1023,0.09181867043177287
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,1,float16,fp8,1023,0.0576853354771932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,1,float16,float16,2047,0.17271467049916586
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,1,float16,fp8,2047,0.10068800052007039
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,1,float16,float16,4095,0.3275093237559001
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,1,float16,fp8,4095,0.17509865760803223
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,1,float16,float16,1,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,1,float16,fp8,1,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,1,float16,float16,3,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,1,float16,fp8,3,0.012538666526476542
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,1,float16,float16,7,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,1,float16,fp8,7,0.01251199965675672
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,1,float16,float16,15,0.012703999876976013
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,1,float16,fp8,15,0.012533333152532578
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,1,float16,float16,31,0.012175999581813812
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,1,float16,fp8,31,0.012373333175977072
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,1,float16,float16,63,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,1,float16,fp8,63,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,1,float16,float16,127,0.012703999876976013
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,1,float16,fp8,127,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,1,float16,float16,255,0.012346666306257248
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,1,float16,fp8,255,0.012554666648308435
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,1,float16,float16,511,0.013045333325862885
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,1,float16,fp8,511,0.012800000607967377
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,1,float16,float16,1023,0.014837333311637243
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,1,float16,fp8,1023,0.014576000471909841
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,1,float16,float16,2047,0.021045332153638203
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,1,float16,fp8,2047,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,1,float16,float16,4095,0.03822399924198786
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,1,float16,fp8,4095,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,1,float16,float16,8191,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,1,float16,fp8,8191,0.03925866633653641
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,1,float16,float16,16383,0.10069333513577779
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,1,float16,fp8,16383,0.060415998101234436
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,1,float16,float16,32767,0.1843199928601583
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,1,float16,fp8,32767,0.10410666465759277
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,1,float16,float16,1,0.05597866574923197
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,1,float16,fp8,1,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,1,float16,float16,3,0.05563733478387197
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,1,float16,fp8,3,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,1,float16,float16,7,0.05597866574923197
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,1,float16,fp8,7,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,1,float16,float16,15,0.05597866574923197
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,1,float16,fp8,15,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,1,float16,float16,31,0.05563733478387197
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,1,float16,float16,63,0.05563733478387197
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,1,float16,fp8,31,0.043354665239652
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,1,float16,fp8,63,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,1,float16,float16,127,0.0576853354771932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,1,float16,fp8,127,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,1,float16,float16,255,0.0576800008614858
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,1,float16,fp8,255,0.04539733131726583
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,1,float16,float16,511,0.09318400422732036
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,1,float16,fp8,511,0.06587733328342438
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,1,float16,float16,1023,0.16743467251459757
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,1,float16,fp8,1023,0.10069333513577779
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,1,float16,fp8,2047,0.1764693260192871
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,1,float16,float16,2047,0.3251146674156189
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,1,float16,float16,1,0.1013866662979126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,1,float16,fp8,1,0.07577600081761678
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,1,float16,float16,3,0.1013759970664978
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,1,float16,fp8,3,0.07612266639868419
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,1,float16,float16,7,0.10069333513577779
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,1,float16,fp8,7,0.07406400144100189
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,1,float16,float16,15,0.1013759970664978
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,1,float16,fp8,15,0.07577600081761678
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,1,float16,float16,31,0.10069333513577779
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,1,float16,fp8,31,0.07611733178297679
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,1,float16,float16,63,0.10103999574979146
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,1,float16,fp8,63,0.07611733178297679
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,1,float16,float16,127,0.10273599624633789
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,1,float16,fp8,127,0.07611733178297679
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,1,float16,float16,255,0.10274133086204529
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,1,float16,fp8,255,0.07611733178297679
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,1,float16,float16,511,0.17066667477289835
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,1,float16,fp8,511,0.11707733074824016
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,1,float16,float16,1,0.1930239995320638
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,1,float16,fp8,1,0.14010666807492575
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,1,float16,float16,3,0.19182932376861572
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,1,float16,fp8,3,0.14148267110188803
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,1,float16,float16,7,0.19063466787338257
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,1,float16,fp8,7,0.1397760013739268
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,1,float16,float16,15,0.19131733973821005
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,1,float16,fp8,15,0.13959999879201254
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,1,float16,float16,31,0.19132266441980997
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,1,float16,fp8,31,0.1397760013739268
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,1,float16,float16,63,0.18944533665974936
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,1,float16,fp8,63,0.14012266198794046
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,1,float16,float16,127,0.19131733973821005
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,1,float16,fp8,127,0.1397760013739268
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,1,float16,float16,255,0.19216533501942953
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,1,float16,float16,1,0.013066666821638743
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,1,float16,fp8,255,0.13961066802342734
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,1,float16,fp8,1,0.012693333129088083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,1,float16,float16,3,0.014565333724021912
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,1,float16,fp8,3,0.012805332740147909
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,1,float16,float16,7,0.014762666076421738
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,1,float16,fp8,7,0.013855999956528345
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,1,float16,float16,15,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,1,float16,fp8,15,0.014570667097965876
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,1,float16,float16,31,0.014762666076421738
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,1,float16,fp8,31,0.012901333471139273
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,1,float16,float16,63,0.014570667097965876
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,1,float16,fp8,63,0.013077333569526672
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,1,float16,float16,127,0.01461333284775416
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,1,float16,fp8,127,0.013994666437307993
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,1,float16,float16,255,0.014560000350077948
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,1,float16,fp8,255,0.012901333471139273
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,1,float16,fp8,511,0.014906667172908783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,1,float16,float16,511,0.01695999999841054
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,1,float16,float16,1023,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,1,float16,fp8,1023,0.01695466662446658
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,1,float16,float16,2047,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,1,float16,float16,4095,0.061434666315714516
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,1,float16,fp8,2047,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,1,float16,fp8,4095,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,1,float16,float16,8191,0.10376532872517903
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,1,float16,fp8,8191,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,1,float16,float16,16383,0.1867093245188395
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,1,float16,fp8,16383,0.10410133004188538
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,1,float16,float16,32767,0.3548159996668498
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,1,float16,fp8,32767,0.19114667177200317
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,1,float16,float16,1,0.36744534969329834
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,1,float16,float16,3,0.36777599652608234
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,1,float16,fp8,1,0.26846933364868164
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,1,float16,fp8,3,0.2677759925524394
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,1,float16,float16,7,0.36813334623972577
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,1,float16,fp8,7,0.26844799518585205
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,1,float16,float16,15,0.36744534969329834
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,1,float16,fp8,15,0.26845866441726685
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,1,float16,float16,31,0.3712000052134196
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,1,float16,fp8,31,0.2681173284848531
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,1,float16,float16,63,0.3688053290049235
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,1,float16,fp8,63,0.26845333973566693
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,1,float16,float16,127,0.3667626778284709
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,1,float16,fp8,127,0.26606933275858563
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,1,float16,float16,1,0.7273759841918945
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,1,float16,fp8,1,0.5189973513285319
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,1,float16,float16,3,0.7232853571573893
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,1,float16,fp8,3,0.5220693349838257
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,1,float16,float16,7,0.7195359865824381
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,1,float16,fp8,7,0.5196799834569296
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,1,float16,float16,15,0.7273813088734945
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,1,float16,fp8,15,0.5183093150456747
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,1,float16,float16,31,0.722266674041748
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,1,float16,fp8,31,0.5210453271865845
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,1,float16,float16,1,0.016666666915019352
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,1,float16,float16,63,0.7205546696980795
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,1,float16,fp8,63,0.5193386475245158
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,1,float16,float16,3,0.01695466662446658
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,1,float16,fp8,1,0.014677333335081736
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,1,float16,fp8,3,0.01462399959564209
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,1,float16,float16,7,0.01695466662446658
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,1,float16,fp8,7,0.014943999548753103
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,1,float16,float16,15,0.01661866654952367
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,1,float16,fp8,15,0.014901333798964819
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,1,float16,float16,31,0.016672000288963318
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,1,float16,fp8,31,0.014762666076421738
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,1,float16,float16,63,0.01695999999841054
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,1,float16,fp8,63,0.014741333822409311
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,1,float16,fp8,127,0.014677333335081736
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,1,float16,float16,127,0.01695466662446658
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,1,float16,float16,255,0.01666133354107539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,1,float16,fp8,255,0.014767999450365702
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,1,float16,float16,511,0.02218666672706604
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,1,float16,fp8,511,0.01870399961868922
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,1,float16,float16,1023,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,1,float16,fp8,1023,0.023557332654794056
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,1,float16,float16,2047,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,1,float16,fp8,2047,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,1,float16,float16,4095,0.10411199927330017
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,1,float16,fp8,4095,0.06382933259010315
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,1,float16,float16,8191,0.18824533621470133
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,1,float16,fp8,8191,0.10546132922172546
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,1,float16,float16,16383,0.35788798332214355
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,1,float16,fp8,16383,0.19079999128977457
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,1,float16,float16,1,0.018757333358128864
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,1,float16,fp8,1,0.016656000167131424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,1,float16,float16,3,0.019007999449968338
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,1,float16,fp8,3,0.016810666769742966
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,1,float16,fp8,7,0.016943999876578648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,1,float16,float16,7,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,1,float16,float16,15,0.019002666076024372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,1,float16,fp8,15,0.01695999999841054
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,1,float16,float16,31,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,1,float16,fp8,31,0.01695466662446658
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,1,float16,float16,63,0.019013332823912304
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,1,float16,fp8,63,0.01666133354107539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,1,float16,float16,127,0.01899733394384384
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,1,float16,fp8,127,0.016965333372354507
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,1,float16,fp8,255,0.017978666971127193
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,1,float16,float16,255,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,1,float16,float16,511,0.02696000039577484
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,1,float16,fp8,511,0.022757334013779957
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,1,float16,float16,1023,0.04607999821503957
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,1,float16,fp8,1023,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,1,float16,float16,2047,0.08225599924723308
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,1,float16,fp8,2047,0.052560001611709595
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,1,float16,float16,4095,0.14728533228238425
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,1,float16,fp8,4095,0.08534399668375652
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,1,float16,float16,8191,0.2752799987792969
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,1,float16,fp8,8191,0.1493333379427592
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,1,float16,float16,1,0.012400000045696894
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,1,float16,fp8,1,0.010858666151762009
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,1,float16,float16,3,0.01181866725285848
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,1,float16,fp8,3,0.011605333536863327
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,1,float16,float16,7,0.012410666793584824
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,1,float16,fp8,7,0.010602666685978571
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,1,float16,float16,15,0.011802667131026586
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,1,float16,fp8,15,0.010741333166758219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,1,float16,float16,31,0.011674666156371435
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,1,float16,fp8,31,0.010826667149861654
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,1,float16,float16,63,0.010570666442314783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,1,float16,fp8,63,0.01251199965675672
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,1,float16,float16,127,0.0124746672809124
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,1,float16,fp8,127,0.010853332777818045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,1,float16,float16,255,0.011674666156371435
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,1,float16,fp8,255,0.012341332932313284
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,1,float16,float16,511,0.011029332876205444
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,1,float16,fp8,511,0.012714666624863943
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,1,float16,float16,1023,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,1,float16,float16,2047,0.01695999999841054
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,1,float16,fp8,1023,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,1,float16,fp8,2047,0.016805333395799
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,1,float16,float16,4095,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,1,float16,fp8,4095,0.016656000167131424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,1,float16,float16,8191,0.02072000006834666
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,1,float16,fp8,8191,0.018661333868900936
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,1,float16,float16,16383,0.02696000039577484
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,1,float16,fp8,16383,0.02277333289384842
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,1,float16,float16,32767,0.046762665112813316
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,1,float16,fp8,32767,0.0314026673634847
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,1,float16,float16,1,0.011882666498422623
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,1,float16,fp8,1,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,1,float16,float16,3,0.011605333536863327
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,1,float16,fp8,3,0.01098666712641716
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,1,float16,float16,7,0.010821333775917688
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,1,float16,fp8,7,0.010847999403874079
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,1,float16,float16,15,0.011920000116030375
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,1,float16,fp8,15,0.012357333054145178
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,1,float16,fp8,31,0.012831999609867731
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,1,float16,float16,31,0.010496000448862711
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,1,float16,float16,63,0.012351999680201212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,1,float16,fp8,63,0.011264000087976456
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,1,float16,float16,127,0.010512000570694605
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,1,float16,fp8,127,0.012586666891972223
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,1,float16,float16,255,0.011887999872366587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,1,float16,fp8,255,0.010645333677530289
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,1,float16,float16,511,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,1,float16,fp8,511,0.01331199953953425
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,1,float16,float16,1023,0.014869333555301031
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,1,float16,fp8,1023,0.015781333049138386
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,1,float16,float16,2047,0.018725333114465077
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,1,float16,float16,4095,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,1,float16,fp8,2047,0.01700266698996226
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,1,float16,fp8,4095,0.019018666197856266
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,1,float16,float16,8191,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,1,float16,fp8,8191,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,1,float16,float16,16383,0.04744533201058706
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,1,float16,fp8,16383,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,1,float16,float16,32767,0.07645866771539052
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,1,float16,float16,1,0.027989332874615986
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,1,float16,fp8,32767,0.04642133414745331
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,1,float16,fp8,1,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,1,float16,float16,3,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,1,float16,fp8,3,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,1,float16,float16,7,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,1,float16,fp8,7,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,1,float16,float16,15,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,1,float16,fp8,15,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,1,float16,float16,31,0.027647999425729115
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,1,float16,fp8,31,0.02309866746266683
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,1,float16,float16,63,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,1,float16,fp8,63,0.022874665757020313
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,1,float16,float16,127,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,1,float16,fp8,127,0.022815999885400135
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,1,float16,float16,255,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,1,float16,fp8,255,0.02276266614596049
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,1,float16,float16,511,0.04539733131726583
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,1,float16,fp8,511,0.03311466674009959
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,1,float16,float16,1023,0.07441066702206929
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,1,float16,fp8,1023,0.04744000236193339
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,1,float16,float16,2047,0.13875200351079306
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,1,float16,fp8,2047,0.08295466502507527
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,1,float16,fp8,4095,0.14097066720326742
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,1,float16,float16,4095,0.2583893338839213
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,1,float16,float16,1,0.012938667088747025
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,1,float16,fp8,1,0.01116266722480456
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,1,float16,float16,3,0.011530666301647821
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,1,float16,fp8,3,0.012869333227475485
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,1,float16,float16,7,0.012533333152532578
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,1,float16,fp8,7,0.012560000022252401
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,1,float16,float16,15,0.013418667018413544
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,1,float16,fp8,15,0.010992000500361124
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,1,float16,float16,31,0.011551999797423681
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,1,float16,fp8,31,0.01201066623131434
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,1,float16,float16,63,0.012351999680201212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,1,float16,fp8,63,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,1,float16,float16,127,0.01309866706530253
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,1,float16,fp8,127,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,1,float16,float16,255,0.011205332974592844
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,1,float16,fp8,255,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,1,float16,float16,511,0.012874666601419449
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,1,float16,fp8,511,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,1,float16,float16,1023,0.014570667097965876
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,1,float16,fp8,1023,0.014736000448465347
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,1,float16,float16,2047,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,1,float16,fp8,2047,0.01905599981546402
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,1,float16,float16,4095,0.03310399999221166
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,1,float16,fp8,4095,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,1,float16,float16,8191,0.05597866574923197
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,1,float16,fp8,8191,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,1,float16,float16,16383,0.0942080020904541
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,1,float16,fp8,16383,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,1,float16,float16,32767,0.17032533884048462
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,1,float16,float16,1,0.04539200166861216
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,1,float16,fp8,1,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,1,float16,fp8,32767,0.10274133086204529
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,1,float16,fp8,3,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,1,float16,float16,3,0.04539733131726583
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,1,float16,float16,7,0.04539733131726583
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,1,float16,fp8,7,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,1,float16,float16,15,0.044031997521718345
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,1,float16,fp8,15,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,1,float16,float16,31,0.04539733131726583
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,1,float16,fp8,31,0.03515200068553289
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,1,float16,float16,63,0.04540266593297323
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,1,float16,fp8,63,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,1,float16,float16,127,0.04539733131726583
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,1,float16,fp8,127,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,1,float16,float16,255,0.04744533201058706
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,1,float16,fp8,255,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,1,float16,float16,511,0.07406933108965556
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,1,float16,fp8,511,0.05358933409055074
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,1,float16,float16,1023,0.12970133622487387
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,1,float16,fp8,1023,0.08021333316961925
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,1,float16,float16,2047,0.2515573302904765
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,1,float16,fp8,2047,0.14147733648618063
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,1,float16,float16,1,0.07816533247629802
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,1,float16,fp8,1,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,1,float16,float16,3,0.07851733267307281
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,1,float16,fp8,3,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,1,float16,float16,7,0.07850666840871175
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,1,float16,float16,63,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,1,float16,fp8,7,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,1,float16,float16,15,0.07850133379300435
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,1,float16,fp8,15,0.0576853354771932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,1,float16,float16,31,0.08021333316961925
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,1,float16,fp8,31,0.059392000238100685
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,1,float16,float16,63,0.07987200220425923
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,1,float16,fp8,63,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,1,float16,float16,127,0.08021333316961925
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,1,float16,fp8,127,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,1,float16,float16,255,0.08055466910203297
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,1,float16,fp8,255,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,1,float16,float16,511,0.13294933239618936
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,1,float16,fp8,511,0.09146133065223694
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,1,float16,float16,1,0.14762666821479797
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,1,float16,fp8,1,0.1088853379090627
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,1,float16,float16,3,0.14729066689809164
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,1,float16,fp8,3,0.1088853379090627
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,1,float16,fp8,7,0.1088853379090627
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,1,float16,float16,7,0.14865066607793173
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,1,float16,float16,15,0.14660267035166422
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,1,float16,fp8,15,0.10922666390736897
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,1,float16,float16,31,0.14779733618100485
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,1,float16,fp8,31,0.10957333445549011
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,1,float16,float16,63,0.14626666903495789
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,1,float16,fp8,63,0.1088853379090627
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,1,float16,float16,127,0.14763200283050537
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,1,float16,fp8,127,0.1088853379090627
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,1,float16,float16,255,0.14831466476122537
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,1,float16,fp8,255,0.10956799983978271
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,1,float16,float16,1,0.014570667097965876
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,1,float16,fp8,1,0.012554666648308435
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,1,float16,float16,3,0.012906666845083237
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,1,float16,fp8,3,0.01302933320403099
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,1,float16,float16,7,0.014565333724021912
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,1,float16,fp8,7,0.012714666624863943
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,1,float16,float16,15,0.012906666845083237
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,1,float16,fp8,15,0.01303999995191892
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,1,float16,float16,31,0.014314666390419006
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,1,float16,fp8,31,0.012629333883523941
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,1,float16,float16,63,0.013248000293970108
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,1,float16,fp8,63,0.013199999928474426
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,1,float16,float16,127,0.012869333227475485
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,1,float16,fp8,127,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,1,float16,float16,255,0.013738666971524557
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,1,float16,fp8,255,0.012693333129088083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,1,float16,float16,511,0.016688000410795212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,1,float16,fp8,511,0.014746667196353277
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,1,float16,float16,1023,0.020080000162124634
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,1,float16,fp8,1023,0.018672000616788864
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,1,float16,float16,2047,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,1,float16,fp8,2047,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,1,float16,float16,4095,0.05870933334032694
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,1,float16,fp8,4095,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,1,float16,float16,8191,0.09865066409111023
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,1,float16,fp8,8191,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,1,float16,fp8,16383,0.10478400190671285
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,1,float16,float16,32767,0.33774932225545246
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,1,float16,fp8,32767,0.1902986764907837
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,1,float16,float16,1,0.2797279953956604
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,1,float16,fp8,1,0.20496533314387003
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,1,float16,float16,3,0.2821120023727417
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,1,float16,fp8,3,0.20497065782546997
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,1,float16,float16,7,0.27938665946324664
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,1,float16,fp8,7,0.2053119937578837
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,1,float16,float16,15,0.2797226707140605
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,1,float16,fp8,15,0.2051360011100769
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,1,float16,float16,31,0.28142933050791424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,1,float16,fp8,31,0.20497065782546997
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,1,float16,float16,63,0.27938133478164673
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,1,float16,fp8,63,0.2053119937578837
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,1,float16,float16,127,0.2797226707140605
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,1,float16,fp8,127,0.20258132616678873
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,1,float16,float16,3,0.5454560120900472
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,1,float16,float16,1,0.5515893300374349
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,1,float16,fp8,1,0.39525866508483887
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,1,float16,fp8,3,0.39458131790161133
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,1,float16,fp8,7,0.39561065038045246
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,1,float16,float16,7,0.5509119828542074
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,1,float16,float16,15,0.5457919836044312
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,1,float16,fp8,15,0.39560532569885254
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,1,float16,float16,31,0.5519359906514486
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,1,float16,fp8,31,0.3945759932200114
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,1,float16,float16,63,0.5478506485621134
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,1,float16,fp8,63,0.3959519863128662
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,1,float16,float16,1,0.014901333798964819
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,1,float16,fp8,1,0.014565333724021912
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,1,float16,float16,3,0.014949332922697067
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,1,float16,fp8,3,0.014639999717473984
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,1,float16,float16,7,0.014912000546852747
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,1,float16,fp8,7,0.014522666732470194
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,1,float16,float16,15,0.014783999572197596
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,1,float16,fp8,15,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,1,float16,float16,31,0.014762666076421738
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,1,float16,fp8,31,0.014560000350077948
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,1,float16,float16,63,0.014720000326633453
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,1,float16,fp8,63,0.014677333335081736
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,1,float16,float16,127,0.014906667172908783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,1,float16,fp8,127,0.014570667097965876
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,1,float16,float16,255,0.014848000059525171
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,1,float16,fp8,255,0.014581333845853806
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,1,float16,float16,511,0.019023999571800232
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,1,float16,fp8,511,0.01666133354107539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,1,float16,float16,1023,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,1,float16,fp8,1023,0.021045332153638203
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,1,float16,float16,2047,0.05120000243186951
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,1,float16,fp8,2047,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,1,float16,float16,4095,0.08260266482830048
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,1,float16,fp8,4095,0.051882664362589516
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,1,float16,float16,8191,0.14574399590492249
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,1,float16,fp8,8191,0.08430932958920796
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,1,float16,float16,16383,0.27153066794077557
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,1,float16,fp8,16383,0.1479626695315043
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,1,float16,float16,1,0.016623999923467636
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,1,float16,float16,3,0.01695466662446658
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,1,float16,fp8,1,0.014959999670584997
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,1,float16,fp8,3,0.014757333944241205
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,1,float16,float16,7,0.01695466662446658
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,1,float16,fp8,7,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,1,float16,float16,15,0.01661866654952367
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,1,float16,fp8,15,0.014965333044528961
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,1,float16,float16,31,0.01672533278663953
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,1,float16,fp8,31,0.014767999450365702
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,1,float16,float16,63,0.017008000363906223
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,1,float16,fp8,63,0.014853333433469137
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,1,float16,float16,127,0.016789333273967106
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,1,float16,fp8,127,0.014762666076421738
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,1,float16,fp8,255,0.014752000570297241
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,1,float16,float16,255,0.016656000167131424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,1,float16,float16,511,0.02082666630546252
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,1,float16,fp8,511,0.018863999595244724
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,1,float16,float16,1023,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,1,float16,fp8,1023,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,1,float16,float16,2047,0.062122667829195656
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,1,float16,fp8,2047,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,1,float16,float16,4095,0.1037546694278717
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,1,float16,fp8,4095,0.06383466720581055
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,1,float16,float16,8191,0.18824533621470133
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,1,float16,fp8,8191,0.10616000493367513
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,1,float16,float16,16383,0.35685332616170246
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,1,float16,fp8,16383,0.19063466787338257
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,1,float16,float16,1,0.012175999581813812
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,1,float16,fp8,1,0.012037333101034164
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,1,float16,float16,3,0.010853332777818045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,1,float16,fp8,3,0.012586666891972223
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,1,float16,float16,7,0.011146667102972666
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,1,float16,fp8,7,0.010682666053374609
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,1,float16,float16,15,0.011157333850860596
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,1,float16,fp8,15,0.01219733307758967
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,1,float16,fp8,31,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,1,float16,float16,31,0.01051733394463857
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,1,float16,float16,63,0.011946666985750198
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,1,float16,fp8,63,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,1,float16,float16,127,0.010847999403874079
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,1,float16,fp8,127,0.012869333227475485
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,1,float16,float16,255,0.011317333827416102
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,1,float16,fp8,255,0.011029332876205444
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,1,float16,float16,511,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,1,float16,fp8,511,0.012383999923865
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,1,float16,float16,1023,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,1,float16,fp8,1023,0.012869333227475485
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,1,float16,float16,2047,0.014757333944241205
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,1,float16,fp8,2047,0.014607999473810196
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,1,float16,float16,4095,0.016949333250522614
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,1,float16,fp8,4095,0.016613333175579708
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,1,float16,float16,8191,0.01871466636657715
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,1,float16,fp8,8191,0.01851733277241389
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,1,float16,float16,16383,0.022863999009132385
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,1,float16,fp8,16383,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,1,float16,float16,32767,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,1,float16,fp8,32767,0.02492266645034154
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,1,float16,float16,65535,0.05597866574923197
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,1,float16,float16,1,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,1,float16,fp8,65535,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,1,float16,float16,3,0.011829332758982977
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,1,float16,fp8,1,0.010693332801262537
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,1,float16,fp8,3,0.011978667229413986
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,1,float16,float16,7,0.011674666156371435
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,1,float16,fp8,7,0.010693332801262537
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,1,float16,float16,15,0.011685332904259363
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,1,float16,fp8,15,0.012373333175977072
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,1,float16,float16,31,0.010677333921194077
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,1,float16,fp8,31,0.011429333438475927
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,1,float16,float16,63,0.012015999605258306
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,1,float16,fp8,63,0.010549332946538925
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,1,float16,float16,127,0.010816000401973724
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,1,float16,fp8,127,0.010805333654085795
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,1,float16,fp8,255,0.012272000312805176
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,1,float16,float16,255,0.010522666076819101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,1,float16,float16,511,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,1,float16,fp8,511,0.013861333330472311
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,1,float16,float16,1023,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,1,float16,fp8,1023,0.012703999876976013
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,1,float16,float16,2047,0.014912000546852747
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,1,float16,fp8,2047,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,1,float16,float16,4095,0.016810666769742966
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,1,float16,fp8,4095,0.014912000546852747
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,1,float16,float16,8191,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,1,float16,fp8,8191,0.018661333868900936
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,1,float16,float16,16383,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,1,float16,fp8,16383,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,1,float16,float16,32767,0.0576853354771932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,1,float16,fp8,32767,0.03719999889532725
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,1,float16,float16,65535,0.09728533029556274
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,1,float16,fp8,65535,0.0576853354771932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,1,float16,float16,1,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,1,float16,fp8,1,0.01870399961868922
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,1,float16,float16,3,0.022863999009132385
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,1,float16,fp8,3,0.019007999449968338
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,1,float16,float16,7,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,1,float16,fp8,7,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,1,float16,float16,15,0.02275199939807256
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,1,float16,float16,31,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,1,float16,fp8,15,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,1,float16,fp8,31,0.018677332748969395
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,1,float16,float16,63,0.022874665757020313
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,1,float16,fp8,63,0.018768000106016796
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,1,float16,float16,127,0.022757334013779957
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,1,float16,fp8,127,0.0186666672428449
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,1,float16,float16,255,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,1,float16,fp8,255,0.019088000059127808
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,1,float16,float16,511,0.03480533262093862
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,1,float16,fp8,511,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,1,float16,float16,1023,0.05358933409055074
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,1,float16,fp8,1023,0.03583999971548716
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,1,float16,float16,2047,0.09659733374913533
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,1,float16,fp8,2047,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,1,float16,float16,4095,0.1747573415438334
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,1,float16,fp8,4095,0.09693866968154907
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,1,float16,float16,8191,0.3264853358268738
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,1,float16,fp8,8191,0.1730560064315796
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,1,float16,float16,1,0.011605333536863327
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,1,float16,float16,3,0.012181332955757776
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,1,float16,fp8,1,0.010960000256697336
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,1,float16,fp8,3,0.010853332777818045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,1,float16,float16,7,0.012351999680201212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,1,float16,fp8,7,0.010816000401973724
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,1,float16,float16,15,0.010687999427318573
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,1,float16,fp8,15,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,1,float16,float16,31,0.012389333297808966
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,1,float16,fp8,31,0.010661333799362183
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,1,float16,float16,63,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,1,float16,fp8,63,0.012666666259368261
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,1,float16,float16,127,0.010672000547250112
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,1,float16,fp8,127,0.01098666712641716
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,1,float16,float16,255,0.01258133351802826
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,1,float16,fp8,255,0.012298667182525
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,1,float16,float16,511,0.012719999998807907
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,1,float16,fp8,511,0.013306666165590286
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,1,float16,fp8,1023,0.014565333724021912
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,1,float16,float16,1023,0.014607999473810196
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,1,float16,float16,2047,0.018858666221300762
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,1,float16,fp8,2047,0.016623999923467636
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,1,float16,float16,4095,0.022175999979178112
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,1,float16,fp8,4095,0.02075733368595441
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,1,float16,float16,8191,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,1,float16,fp8,8191,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,1,float16,float16,16383,0.059061333537101746
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,1,float16,fp8,16383,0.039594667653242745
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,1,float16,float16,32767,0.09966933727264404
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,1,float16,fp8,32767,0.059861332178115845
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,1,float16,float16,65535,0.18107734123865762
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,1,float16,float16,1,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,1,float16,fp8,65535,0.09865066409111023
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,1,float16,fp8,1,0.027306665976842243
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,1,float16,float16,3,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,1,float16,fp8,3,0.02697066714366277
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,1,float16,float16,7,0.03310399999221166
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,1,float16,fp8,7,0.02696000039577484
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,1,float16,float16,15,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,1,float16,fp8,15,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,1,float16,float16,31,0.03310399999221166
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,1,float16,fp8,31,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,1,float16,float16,63,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,1,float16,fp8,63,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,1,float16,float16,127,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,1,float16,fp8,127,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,1,float16,float16,255,0.03549866626660029
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,1,float16,fp8,255,0.02696000039577484
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,1,float16,float16,511,0.053930665055910744
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,1,float16,fp8,511,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,1,float16,fp8,1023,0.0580266664425532
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,1,float16,float16,1023,0.09250133236249287
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,1,float16,float16,2047,0.17459734280904135
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,1,float16,fp8,2047,0.10068800052007039
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,1,float16,float16,4095,0.3278506596883138
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,1,float16,float16,1,0.05597866574923197
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,1,float16,fp8,4095,0.17629865805308023
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,1,float16,fp8,1,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,1,float16,float16,3,0.056314667065938316
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,1,float16,fp8,3,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,1,float16,float16,7,0.05563733478387197
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,1,float16,fp8,7,0.043354665239652
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,1,float16,float16,15,0.05563733478387197
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,1,float16,fp8,15,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,1,float16,float16,31,0.05563733478387197
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,1,float16,fp8,31,0.043354665239652
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,1,float16,float16,63,0.05563733478387197
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,1,float16,fp8,63,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,1,float16,float16,127,0.0576853354771932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,1,float16,fp8,127,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,1,float16,float16,255,0.0576853354771932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,1,float16,fp8,255,0.04539733131726583
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,1,float16,float16,511,0.09454933802286784
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,1,float16,fp8,511,0.06587733328342438
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,1,float16,float16,1023,0.16709333658218384
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,1,float16,fp8,1023,0.10069333513577779
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,1,float16,float16,1,0.10274133086204529
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,1,float16,fp8,1,0.07611733178297679
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,1,float16,float16,3,0.10274666547775269
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,1,float16,fp8,3,0.07408000032107036
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,1,float16,float16,7,0.10273599624633789
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,1,float16,fp8,7,0.07611733178297679
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,1,float16,float16,15,0.10274133086204529
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,1,float16,fp8,15,0.07611733178297679
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,1,float16,float16,31,0.10274133086204529
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,1,float16,fp8,31,0.07611733178297679
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,1,float16,float16,63,0.10205866893132527
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,1,float16,fp8,63,0.07406933108965556
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,1,float16,float16,127,0.10240532954533894
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,1,float16,fp8,127,0.07645333309968312
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,1,float16,fp8,255,0.07611200213432312
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,1,float16,float16,255,0.10274133086204529
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,1,float16,float16,511,0.17305066188176474
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,1,float16,float16,1,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,1,float16,fp8,511,0.11742400129636128
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,1,float16,fp8,1,0.013834666460752487
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,1,float16,float16,3,0.01102399950226148
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,1,float16,fp8,3,0.012597333639860153
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,1,float16,float16,7,0.013445333888133367
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,1,float16,fp8,7,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,1,float16,float16,15,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,1,float16,fp8,15,0.012448000411192576
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,1,float16,fp8,31,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,1,float16,float16,31,0.012448000411192576
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,1,float16,float16,63,0.01423466702302297
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,1,float16,fp8,63,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,1,float16,float16,127,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,1,float16,fp8,127,0.013797332843144735
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,1,float16,float16,255,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,1,float16,fp8,255,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,1,float16,float16,511,0.01303999995191892
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,1,float16,fp8,511,0.012863999853531519
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,1,float16,float16,1023,0.014858666807413101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,1,float16,fp8,1023,0.014741333822409311
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,1,float16,float16,2047,0.02275199939807256
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,1,float16,fp8,2047,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,1,float16,float16,4095,0.038912000755469
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,1,float16,fp8,4095,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,1,float16,float16,8191,0.059392000238100685
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,1,float16,fp8,8191,0.040618665516376495
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,1,float16,float16,16383,0.1013759970664978
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,1,float16,fp8,16383,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,1,float16,float16,32767,0.1838080088297526
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,1,float16,fp8,32767,0.10308266679445903
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,1,float16,float16,65535,0.3490133285522461
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,1,float16,fp8,65535,0.18875734011332193
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,1,float16,float16,1,0.19183466831843057
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,1,float16,fp8,1,0.14148267110188803
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,1,float16,float16,3,0.19234132766723633
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,1,float16,fp8,3,0.1397760013739268
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,1,float16,float16,7,0.19285333156585693
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,1,float16,fp8,7,0.14131733775138855
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,1,float16,float16,15,0.19131733973821005
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,1,float16,float16,31,0.19285333156585693
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,1,float16,fp8,15,0.14165332913398743
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,1,float16,fp8,31,0.14148267110188803
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,1,float16,float16,63,0.1930239995320638
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,1,float16,float16,127,0.19080533583958945
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,1,float16,fp8,63,0.14165332913398743
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,1,float16,fp8,127,0.14148267110188803
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,1,float16,float16,255,0.1930239995320638
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,1,float16,fp8,255,0.14148267110188803
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,1,float16,fp8,1,0.2691413362820943
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,1,float16,float16,1,0.3722240130106608
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,1,float16,float16,3,0.3729066848754883
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,1,float16,fp8,3,0.2688000003496806
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,1,float16,float16,7,0.37120532989501953
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,1,float16,fp8,7,0.2701653242111206
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,1,float16,float16,15,0.36983998616536456
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,1,float16,fp8,15,0.2691413362820943
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,1,float16,float16,31,0.3722240130106608
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,1,float16,fp8,31,0.26846933364868164
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,1,float16,float16,63,0.3722240130106608
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,1,float16,fp8,63,0.2688000003496806
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,1,float16,float16,1,0.01461333284775416
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,1,float16,float16,127,0.3691519896189372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,1,float16,fp8,127,0.2674400011698405
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,1,float16,fp8,1,0.012560000022252401
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,1,float16,float16,16383,0.17799999316533408
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,1,float16,float16,3,0.012853333105643591
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,1,float16,fp8,3,0.01302933320403099
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,1,float16,float16,7,0.01461333284775416
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,1,float16,fp8,7,0.012901333471139273
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,1,float16,float16,15,0.012901333471139273
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,1,float16,fp8,15,0.013130666067202887
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,1,float16,float16,31,0.01461333284775416
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,1,float16,fp8,31,0.012736000120639801
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,1,float16,float16,63,0.014762666076421738
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,1,float16,fp8,63,0.013023999830087027
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,1,float16,float16,127,0.014277332772811254
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,1,float16,fp8,127,0.012730666746695837
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,1,float16,float16,255,0.012863999853531519
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,1,float16,fp8,255,0.01482133318980535
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,1,float16,float16,511,0.01666133354107539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,1,float16,fp8,511,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,1,float16,float16,1023,0.021013334393501282
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,1,float16,fp8,1023,0.018672000616788864
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,1,float16,float16,2047,0.039594667653242745
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,1,float16,fp8,2047,0.025936000049114227
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,1,float16,float16,4095,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,1,float16,fp8,4095,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,1,float16,float16,8191,0.10410666465759277
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,1,float16,fp8,8191,0.06382399797439575
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,1,float16,float16,16383,0.1879040002822876
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,1,float16,fp8,16383,0.10479467113812764
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,1,float16,float16,32767,0.35447466373443604
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,1,float16,fp8,32767,0.19063466787338257
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,1,float16,float16,1,0.014917333920796713
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,1,float16,fp8,1,0.014570667097965876
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,1,float16,float16,3,0.014762666076421738
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,1,float16,float16,7,0.014943999548753103
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,1,float16,fp8,3,0.014511999984582266
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,1,float16,fp8,7,0.014741333822409311
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,1,float16,float16,15,0.014757333944241205
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,1,float16,fp8,15,0.014671999961137772
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,1,float16,float16,31,0.014949332922697067
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,1,float16,fp8,31,0.014570667097965876
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,1,float16,float16,63,0.014629332969586054
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,1,float16,fp8,63,0.014677333335081736
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,1,float16,float16,127,0.014762666076421738
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,1,float16,fp8,127,0.014570667097965876
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,1,float16,float16,255,0.014757333944241205
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,1,float16,fp8,255,0.013653332988421122
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,1,float16,float16,511,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,1,float16,fp8,511,0.016672000288963318
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,1,float16,float16,1023,0.027306665976842243
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,1,float16,fp8,1023,0.02075733368595441
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,1,float16,float16,2047,0.051551997661590576
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,1,float16,fp8,2047,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,1,float16,float16,4095,0.08260266482830048
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,1,float16,fp8,4095,0.0532533327738444
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,1,float16,float16,8191,0.14523733655611673
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,1,float16,fp8,8191,0.0846506655216217
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,1,float16,float16,16383,0.2725546757380168
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,1,float16,fp8,16383,0.14780267079671225
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,1,float16,float16,1,0.011781333635250727
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,1,float16,fp8,1,0.012351999680201212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,1,float16,float16,3,0.011317333827416102
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,1,float16,fp8,3,0.01292266696691513
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,1,float16,float16,7,0.010687999427318573
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,1,float16,fp8,7,0.013925333817799887
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,1,float16,float16,15,0.011002667248249054
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,1,float16,fp8,15,0.012234666695197424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,1,float16,float16,31,0.010858666151762009
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,1,float16,float16,63,0.011674666156371435
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,1,float16,fp8,31,0.012005332857370377
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,1,float16,fp8,63,0.01227733368674914
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,1,float16,float16,127,0.010842667271693548
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,1,float16,fp8,127,0.012693333129088083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,1,float16,float16,255,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,1,float16,fp8,255,0.012154666086037954
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,1,float16,float16,511,0.011424000064531961
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,1,float16,fp8,511,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,1,float16,float16,1023,0.01251199965675672
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,1,float16,fp8,1023,0.012560000022252401
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,1,float16,fp8,2047,0.012357333054145178
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,1,float16,float16,2047,0.012810666114091873
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,1,float16,float16,4095,0.01695466662446658
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,1,float16,fp8,4095,0.01666133354107539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,1,float16,float16,8191,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,1,float16,fp8,8191,0.01666133354107539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,1,float16,float16,16383,0.021055998901526134
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,1,float16,fp8,16383,0.018709332992633183
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,1,float16,float16,32767,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,1,float16,fp8,32767,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,1,float16,float16,65535,0.047450666626294456
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,1,float16,fp8,65535,0.0314026673634847
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,1,float16,float16,1,0.010821333775917688
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,1,float16,fp8,1,0.010853332777818045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,1,float16,float16,3,0.012223999947309494
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,1,float16,fp8,3,0.012202666451533636
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,1,float16,float16,7,0.011338666081428528
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,1,float16,fp8,7,0.011205332974592844
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,1,float16,float16,15,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,1,float16,fp8,15,0.012213333199421564
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,1,float16,float16,31,0.011328000575304031
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,1,float16,float16,63,0.01313599944114685
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,1,float16,fp8,31,0.011786667009194693
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,1,float16,float16,127,0.010832000523805618
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,1,float16,fp8,63,0.012533333152532578
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,1,float16,fp8,127,0.012789333860079447
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,1,float16,float16,255,0.01051733394463857
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,1,float16,fp8,255,0.012362666428089142
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,1,float16,float16,511,0.012624000509579977
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,1,float16,fp8,511,0.013770667215188345
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,1,float16,float16,1023,0.012629333883523941
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,1,float16,fp8,1023,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,1,float16,float16,2047,0.01699200024207433
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,1,float16,fp8,2047,0.017717332889636356
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,1,float16,float16,4095,0.018432000031073887
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,1,float16,fp8,4095,0.018709332992633183
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,1,float16,float16,8191,0.02109333376089732
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,1,float16,fp8,8191,0.02070933332045873
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,1,float16,float16,16383,0.028666667640209198
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,1,float16,fp8,16383,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,1,float16,float16,32767,0.049498667319615684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,1,float16,fp8,32767,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,1,float16,fp8,65535,0.04744533201058706
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,1,float16,float16,65535,0.07679999868075053
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,1,float16,float16,1,0.01904533306757609
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,1,float16,fp8,1,0.016677333662907284
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,1,float16,float16,3,0.019002666076024372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,1,float16,fp8,3,0.01666133354107539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,1,float16,float16,7,0.01870399961868922
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,1,float16,fp8,7,0.016789333273967106
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,1,float16,float16,15,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,1,float16,fp8,15,0.016650666793187458
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,1,float16,float16,31,0.018709332992633183
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,1,float16,fp8,31,0.01681600014368693
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,1,float16,float16,63,0.01911466692884763
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,1,float16,fp8,63,0.016800000021855038
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,1,float16,float16,127,0.018709332992633183
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,1,float16,fp8,127,0.016810666769742966
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,1,float16,float16,255,0.019797333826621372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,1,float16,float16,511,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,1,float16,fp8,255,0.01666133354107539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,1,float16,fp8,511,0.02276800076166789
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,1,float16,float16,1023,0.0460746685663859
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,1,float16,fp8,1023,0.03105599929889043
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,1,float16,float16,2047,0.08260266482830048
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,1,float16,fp8,2047,0.05359466870625814
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,1,float16,float16,4095,0.14711999893188477
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,1,float16,fp8,4095,0.08636266986529033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,1,float16,float16,8191,0.274944007396698
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,1,float16,float16,1,0.010970667004585266
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,1,float16,fp8,8191,0.1495039959748586
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,1,float16,fp8,1,0.012063999970753988
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,1,float16,float16,3,0.010693332801262537
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,1,float16,fp8,3,0.012549333274364471
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,1,float16,float16,7,0.013141332815090815
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,1,float16,fp8,7,0.012624000509579977
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,1,float16,fp8,15,0.01099733387430509
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,1,float16,float16,15,0.010682666053374609
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,1,float16,float16,31,0.012005332857370377
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,1,float16,fp8,31,0.011968000481526056
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,1,float16,float16,63,0.012159999459981918
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,1,float16,fp8,63,0.010687999427318573
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,1,float16,float16,127,0.01201066623131434
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,1,float16,fp8,127,0.012223999947309494
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,1,float16,float16,255,0.011850666254758835
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,1,float16,fp8,255,0.013114667187134424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,1,float16,float16,511,0.013141332815090815
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,1,float16,fp8,511,0.012389333297808966
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,1,float16,float16,1023,0.014762666076421738
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,1,float16,fp8,1023,0.01461333284775416
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,1,float16,float16,2047,0.01871466636657715
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,1,float16,fp8,2047,0.016901332885026932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,1,float16,float16,4095,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,1,float16,fp8,4095,0.020741333564122517
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,1,float16,float16,8191,0.028677334388097126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,1,float16,fp8,8191,0.02347733328739802
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,1,float16,float16,16383,0.051216001311937966
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,1,float16,fp8,16383,0.034485332667827606
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,1,float16,float16,32767,0.07816533247629802
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,1,float16,fp8,32767,0.0484746644894282
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,1,float16,float16,65535,0.13806933164596558
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,1,float16,float16,1,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,1,float16,fp8,65535,0.07611733178297679
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,1,float16,fp8,1,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,1,float16,float16,3,0.027653334041436512
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,1,float16,fp8,3,0.024127999941507976
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,1,float16,float16,7,0.027642667293548584
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,1,float16,fp8,7,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,1,float16,fp8,31,0.022810667753219604
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,1,float16,fp8,15,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,1,float16,float16,63,0.027306665976842243
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,1,float16,fp8,63,0.023546665906906128
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,1,float16,float16,127,0.027295999228954315
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,1,float16,fp8,127,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,1,float16,float16,255,0.02934933453798294
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,1,float16,fp8,255,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,1,float16,float16,511,0.04471466441949209
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,1,float16,fp8,511,0.03311466674009959
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,1,float16,fp8,1023,0.04949333270390829
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,1,float16,float16,1023,0.07338666419188182
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,1,float16,fp8,2047,0.08260266482830048
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,1,float16,float16,2047,0.13755733768145242
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,1,float16,fp8,4095,0.14165332913398743
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,1,float16,float16,4095,0.2583893338839213
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,1,float16,float16,1,0.04539200166861216
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,1,float16,fp8,1,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,1,float16,float16,3,0.04539733131726583
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,1,float16,fp8,3,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,1,float16,float16,7,0.04539733131726583
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,1,float16,float16,15,0.04539733131726583
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,1,float16,fp8,15,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,1,float16,float16,31,0.04540266593297323
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,1,float16,fp8,31,0.035162667433420815
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,1,float16,float16,63,0.04539733131726583
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,1,float16,fp8,63,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,1,float16,float16,127,0.04539733131726583
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,1,float16,fp8,127,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,1,float16,float16,255,0.04744533201058706
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,1,float16,fp8,255,0.036864000062147774
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,1,float16,float16,511,0.0744053324063619
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,1,float16,fp8,511,0.05358933409055074
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,1,float16,float16,1023,0.13038399815559387
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,1,float16,fp8,1023,0.08021866778532664
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,1,float16,float16,1,0.07884799937407176
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,1,float16,fp8,1,0.059392000238100685
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,1,float16,float16,3,0.07850666840871175
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,1,float16,fp8,3,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,1,float16,float16,7,0.08021333316961925
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,1,float16,fp8,7,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,1,float16,float16,15,0.08021333316961925
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,1,float16,fp8,15,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,1,float16,float16,31,0.07952000200748444
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,1,float16,fp8,31,0.0580266664425532
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,1,float16,float16,63,0.08021333316961925
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,1,float16,fp8,63,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,1,float16,float16,127,0.07987200220425923
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,1,float16,fp8,127,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,1,float16,float16,255,0.08057066798210144
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,1,float16,fp8,255,0.06041066845258077
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,1,float16,float16,511,0.13210133711496988
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,1,float16,fp8,511,0.09079466263453166
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,1,float16,float16,1,0.01098666712641716
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,1,float16,fp8,1,0.012554666648308435
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,1,float16,float16,3,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,1,float16,fp8,3,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,1,float16,float16,7,0.01202133297920227
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,1,float16,fp8,7,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,1,float16,float16,15,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,1,float16,fp8,15,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,1,float16,float16,31,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,1,float16,fp8,31,0.012346666306257248
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,1,float16,float16,63,0.011839999506870905
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,1,float16,fp8,63,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,1,float16,float16,127,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,1,float16,fp8,127,0.01249066616098086
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,1,float16,float16,255,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,1,float16,fp8,255,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,1,float16,float16,511,0.012858666479587555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,1,float16,fp8,511,0.01251199965675672
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,1,float16,float16,1023,0.014922666052977243
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,1,float16,fp8,1023,0.014767999450365702
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,1,float16,float16,2047,0.0210506667693456
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,1,float16,fp8,2047,0.019002666076024372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,1,float16,float16,4095,0.031066666046778362
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,1,float16,fp8,4095,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,1,float16,float16,8191,0.056320001681645714
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,1,float16,fp8,8191,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,1,float16,float16,16383,0.09523199995358785
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,1,float16,fp8,16383,0.06041066845258077
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,1,float16,float16,32767,0.17151999473571777
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,1,float16,fp8,32767,0.10274133086204529
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,1,float16,float16,65535,0.32341333230336505
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,1,float16,fp8,65535,0.18876266479492188
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,1,float16,float16,1,0.14779733618100485
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,1,float16,fp8,1,0.10956799983978271
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,1,float16,float16,3,0.14762666821479797
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,1,float16,fp8,3,0.10957866907119751
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,1,float16,float16,7,0.14865066607793173
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,1,float16,fp8,7,0.10921600461006165
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,1,float16,float16,15,0.1493333379427592
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,1,float16,fp8,15,0.10923199852307637
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,1,float16,float16,31,0.14779733618100485
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,1,float16,fp8,31,0.10922666390736897
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,1,float16,float16,63,0.14779733618100485
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,1,float16,fp8,63,0.1088853379090627
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,1,float16,float16,127,0.14779733618100485
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,1,float16,fp8,127,0.10992532968521118
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,1,float16,float16,255,0.15001599987347922
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,1,float16,fp8,255,0.11059199770291646
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,1,float16,float16,1,0.28279467423756915
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,1,float16,fp8,1,0.2051360011100769
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,1,float16,float16,3,0.2821013331413269
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,1,float16,fp8,3,0.2053119937578837
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,1,float16,float16,7,0.2827999989191691
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,1,float16,fp8,7,0.20718934138615927
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,1,float16,float16,15,0.28245333830515545
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,1,float16,fp8,15,0.2053119937578837
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,1,float16,float16,31,0.283135990301768
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,1,float16,fp8,31,0.20599466562271118
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,1,float16,float16,63,0.2834773262341817
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,1,float16,fp8,63,0.2058239976565043
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,1,float16,float16,127,0.28142933050791424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,1,float16,fp8,127,0.2034346659978231
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,1,float16,float16,1,0.012858666479587555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,1,float16,fp8,1,0.012858666479587555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,1,float16,float16,3,0.012655999511480331
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,1,float16,fp8,3,0.012741333494583765
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,1,float16,float16,7,0.0144213338692983
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,1,float16,fp8,7,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,1,float16,float16,15,0.014671999961137772
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,1,float16,fp8,15,0.012863999853531519
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,1,float16,float16,31,0.012629333883523941
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,1,float16,fp8,31,0.012725333372751871
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,1,float16,float16,63,0.012917333592971167
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,1,float16,fp8,63,0.01259200026591619
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,1,float16,float16,127,0.012863999853531519
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,1,float16,fp8,127,0.012917333592971167
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,1,float16,float16,255,0.014565333724021912
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,1,float16,fp8,255,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,1,float16,float16,511,0.016607999801635742
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,1,float16,fp8,511,0.014805333067973455
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,1,float16,float16,1023,0.01899733394384384
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,1,float16,fp8,1023,0.016965333372354507
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,1,float16,float16,2047,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,1,float16,fp8,2047,0.025258667767047882
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,1,float16,float16,4095,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,1,float16,fp8,4095,0.03889599939187368
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,1,float16,float16,8191,0.09830400347709656
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,1,float16,fp8,8191,0.06178666651248932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,1,float16,float16,16383,0.17834667364756265
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,1,float16,fp8,16383,0.10478933652242024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,1,float16,float16,32767,0.33774932225545246
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,1,float16,float16,31,0.027295999228954315
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,1,float16,fp8,32767,0.19166400035222372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,1,float16,float16,1,0.014752000570297241
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,1,float16,fp8,1,0.012693333129088083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,1,float16,float16,3,0.012810666114091873
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,1,float16,fp8,3,0.013034666577974955
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,1,float16,float16,7,0.012805332740147909
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,1,float16,fp8,7,0.013306666165590286
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,1,float16,float16,15,0.01441066712141037
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,1,float16,fp8,15,0.014570667097965876
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,1,float16,float16,31,0.012709333250919977
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,1,float16,fp8,31,0.013536000003417334
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,1,float16,float16,63,0.014511999984582266
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,1,float16,fp8,63,0.01302933320403099
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,1,float16,float16,127,0.01441066712141037
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,1,float16,fp8,127,0.014218666901191076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,1,float16,float16,255,0.014261333892742792
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,1,float16,fp8,255,0.012863999853531519
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,1,float16,float16,511,0.016656000167131424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,1,float16,fp8,511,0.014576000471909841
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,1,float16,float16,1023,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,1,float16,fp8,1023,0.01729600007335345
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,1,float16,float16,2047,0.04027733455101649
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,1,float16,fp8,7,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,1,float16,fp8,2047,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,1,float16,float16,4095,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,1,float16,fp8,4095,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,1,float16,float16,8191,0.10547199845314026
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,1,float16,fp8,8191,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,1,float16,float16,16383,0.18859734137852988
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,1,float16,fp8,16383,0.10615467031796773
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,1,float16,float16,32767,0.35789334774017334
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,1,float16,fp8,32767,0.19131199518839517
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,1,float16,float16,1,0.012330666184425354
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,1,float16,fp8,1,0.010687999427318573
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,1,float16,float16,3,0.010687999427318573
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,1,float16,fp8,3,0.012351999680201212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,1,float16,fp8,7,0.010858666151762009
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,1,float16,float16,7,0.01051733394463857
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,1,float16,float16,15,0.011264000087976456
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,1,float16,fp8,15,0.011887999872366587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,1,float16,fp8,31,0.012346666306257248
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,1,float16,float16,31,0.010522666076819101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,1,float16,float16,63,0.011605333536863327
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,1,float16,fp8,63,0.011328000575304031
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,1,float16,float16,127,0.010512000570694605
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,1,float16,fp8,127,0.011546666423479715
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,1,float16,float16,255,0.01181866725285848
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,1,float16,fp8,255,0.011551999797423681
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,1,float16,float16,511,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,1,float16,fp8,511,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,1,float16,float16,1023,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,1,float16,fp8,1023,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,1,float16,float16,2047,0.012319999436537424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,1,float16,fp8,2047,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,1,float16,float16,4095,0.016469333320856094
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,1,float16,fp8,4095,0.016607999801635742
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,1,float16,float16,8191,0.018709332992633183
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,1,float16,fp8,8191,0.01870399961868922
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,1,float16,float16,16383,0.02081599955757459
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,1,float16,fp8,16383,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,1,float16,float16,32767,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,1,float16,fp8,32767,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,1,float16,float16,65535,0.03618133316437403
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,1,float16,fp8,65535,0.0266239990790685
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,1,float16,float16,131071,0.060415998101234436
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,1,float16,fp8,131071,0.040618665516376495
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,1,float16,float16,1,0.011605333536863327
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,1,float16,fp8,1,0.01221866657336553
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,1,float16,float16,3,0.011034666250149408
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,1,float16,fp8,3,0.012015999605258306
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,1,float16,float16,7,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,1,float16,fp8,7,0.011018666128317514
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,1,float16,float16,15,0.01051733394463857
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,1,float16,fp8,15,0.012272000312805176
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,1,float16,float16,31,0.010682666053374609
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,1,float16,fp8,31,0.011039999624093374
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,1,float16,float16,63,0.012069333344697952
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,1,float16,fp8,63,0.010687999427318573
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,1,float16,float16,127,0.010853332777818045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,1,float16,fp8,127,0.01191466674208641
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,1,float16,fp8,255,0.010863999525705973
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,1,float16,float16,255,0.010522666076819101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,1,float16,float16,511,0.012175999581813812
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,1,float16,fp8,511,0.012400000045696894
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,1,float16,float16,1023,0.012351999680201212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,1,float16,fp8,1023,0.012346666306257248
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,1,float16,float16,2047,0.016384000579516094
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,1,float16,fp8,2047,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,1,float16,float16,4095,0.01681600014368693
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,1,float16,fp8,4095,0.016997333616018295
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,1,float16,float16,8191,0.018709332992633183
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,1,float16,fp8,8191,0.018853332847356796
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,1,float16,float16,16383,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,1,float16,fp8,16383,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,1,float16,float16,32767,0.03719999889532725
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,1,float16,fp8,32767,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,1,float16,float16,65535,0.05769066512584686
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,1,float16,fp8,65535,0.03721066564321518
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,1,float16,float16,131071,0.09898666540781657
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,1,float16,float16,1,0.01681600014368693
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,1,float16,fp8,131071,0.05905066430568695
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,1,float16,fp8,1,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,1,float16,float16,3,0.016805333395799
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,1,float16,fp8,3,0.014949332922697067
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,1,float16,float16,7,0.016895999511082966
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,1,float16,fp8,7,0.014912000546852747
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,1,float16,float16,15,0.016666666915019352
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,1,float16,fp8,15,0.014949332922697067
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,1,float16,float16,31,0.016805333395799
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,1,float16,fp8,31,0.014954666296641031
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,1,float16,float16,63,0.016650666793187458
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,1,float16,fp8,63,0.014901333798964819
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,1,float16,float16,127,0.016672000288963318
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,1,float16,fp8,127,0.014848000059525171
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,1,float16,float16,255,0.016623999923467636
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,1,float16,fp8,255,0.014671999961137772
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,1,float16,float16,511,0.02109333376089732
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,1,float16,fp8,511,0.01865600049495697
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,1,float16,float16,1023,0.034815999368826546
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,1,float16,fp8,1023,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,1,float16,float16,2047,0.061792001128196716
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,1,float16,fp8,2047,0.041296000281969704
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,1,float16,float16,4095,0.10478933652242024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,1,float16,fp8,4095,0.06382399797439575
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,1,float16,float16,8191,0.18893865744272867
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,1,float16,fp8,8191,0.10615467031796773
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,1,float16,float16,16383,0.3599413235982259
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,1,float16,float16,1,0.012144000579913458
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,1,float16,fp8,1,0.012346666306257248
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,1,float16,fp8,16383,0.19080533583958945
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,1,float16,float16,3,0.011936000237862269
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,1,float16,fp8,3,0.012069333344697952
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,1,float16,float16,7,0.010858666151762009
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,1,float16,fp8,7,0.011952000359694162
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,1,float16,float16,15,0.012005332857370377
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,1,float16,fp8,15,0.01110400011142095
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,1,float16,float16,31,0.011946666985750198
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,1,float16,fp8,31,0.011776000261306763
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,1,float16,float16,63,0.01119999960064888
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,1,float16,fp8,63,0.011776000261306763
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,1,float16,float16,127,0.012063999970753988
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,1,float16,fp8,127,0.010682666053374609
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,1,float16,float16,255,0.011264000087976456
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,1,float16,fp8,255,0.011893333246310553
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,1,float16,float16,511,0.012736000120639801
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,1,float16,fp8,511,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,1,float16,float16,1023,0.013429333766301474
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,1,float16,fp8,1023,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,1,float16,float16,2047,0.01655999943614006
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,1,float16,fp8,2047,0.016469333320856094
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,1,float16,float16,4095,0.016656000167131424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,1,float16,fp8,4095,0.016800000021855038
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,1,float16,float16,8191,0.02276266614596049
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,1,float16,fp8,8191,0.01870399961868922
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,1,float16,float16,16383,0.03754666695992152
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,1,float16,fp8,16383,0.024911999702453613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,1,float16,float16,32767,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,1,float16,fp8,32767,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,1,float16,float16,65535,0.10240000486373901
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,1,float16,fp8,65535,0.05836800237496694
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,1,float16,float16,131071,0.18551466862360635
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,1,float16,fp8,131071,0.1013706624507904
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,1,float16,float16,1,0.022800001005331676
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,1,float16,fp8,1,0.01883200059334437
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,1,float16,float16,3,0.02295999974012375
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,1,float16,fp8,3,0.018768000106016796
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,1,float16,float16,7,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,1,float16,fp8,7,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,1,float16,float16,15,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,1,float16,fp8,15,0.01870399961868922
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,1,float16,float16,31,0.02276266614596049
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,1,float16,fp8,31,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,1,float16,float16,63,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,1,float16,fp8,63,0.019002666076024372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,1,float16,fp8,127,0.018719999740521114
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,1,float16,float16,127,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,1,float16,float16,255,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,1,float16,fp8,255,0.018698666244745255
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,1,float16,float16,511,0.034128000338872276
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,1,float16,fp8,511,0.025146665672461193
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,1,float16,float16,1023,0.05358933409055074
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,1,float16,fp8,1023,0.03549866626660029
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,1,float16,float16,2047,0.09898666540781657
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,1,float16,fp8,2047,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,1,float16,float16,4095,0.17561600605646768
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,1,float16,fp8,4095,0.09864532947540283
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,1,float16,float16,8191,0.3319466710090637
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,1,float16,fp8,8191,0.17254400253295898
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,1,float16,float16,1,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,1,float16,fp8,1,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,1,float16,fp8,3,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,1,float16,float16,3,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,1,float16,float16,7,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,1,float16,fp8,7,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,1,float16,float16,15,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,1,float16,fp8,15,0.02696000039577484
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,1,float16,float16,31,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,1,float16,fp8,31,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,1,float16,float16,63,0.03310399999221166
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,1,float16,fp8,63,0.02697066714366277
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,1,float16,float16,127,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,1,float16,fp8,127,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,1,float16,float16,255,0.03549866626660029
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,1,float16,fp8,255,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,1,float16,float16,511,0.05563733478387197
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,1,float16,fp8,511,0.04095999896526337
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,1,float16,float16,1023,0.09317866961161296
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,1,float16,fp8,1023,0.059392000238100685
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,1,float16,float16,2047,0.17629865805308023
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,1,float16,fp8,2047,0.1013706624507904
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,1,float16,float16,1,0.05563733478387197
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,1,float16,fp8,1,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,1,float16,float16,3,0.05563733478387197
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,1,float16,fp8,3,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,1,float16,float16,7,0.05563200016816457
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,1,float16,fp8,7,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,1,float16,float16,15,0.05563733478387197
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,1,float16,fp8,15,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,1,float16,float16,31,0.055973331133524575
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,1,float16,fp8,31,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,1,float16,float16,63,0.05563733478387197
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,1,float16,fp8,63,0.04334400097529093
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,1,float16,float16,127,0.0576853354771932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,1,float16,fp8,127,0.04334400097529093
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,1,float16,float16,255,0.05836800237496694
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,1,float16,fp8,255,0.04539200166861216
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,1,float16,float16,511,0.09454400340716045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,1,float16,fp8,511,0.06587733328342438
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,1,float16,float16,1023,0.1704960068066915
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,1,float16,float16,1,0.012138667205969492
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,1,float16,fp8,1023,0.10069333513577779
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,1,float16,fp8,1,0.01227733368674914
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,1,float16,float16,3,0.010682666053374609
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,1,float16,fp8,3,0.01258133351802826
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,1,float16,float16,7,0.012293333808581034
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,1,float16,fp8,7,0.010858666151762009
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,1,float16,float16,15,0.012442667037248611
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,1,float16,fp8,15,0.011882666498422623
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,1,float16,float16,31,0.011034666250149408
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,1,float16,fp8,31,0.012186666329701742
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,1,float16,float16,63,0.01109333336353302
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,1,float16,fp8,63,0.010853332777818045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,1,float16,float16,127,0.012181332955757776
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,1,float16,fp8,127,0.011029332876205444
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,1,float16,float16,255,0.010687999427318573
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,1,float16,fp8,255,0.012453333785136541
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,1,float16,float16,511,0.012794667234023413
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,1,float16,fp8,511,0.013807999591032663
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,1,float16,float16,1023,0.01440000037352244
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,1,float16,fp8,1023,0.012736000120639801
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,1,float16,float16,2047,0.01666133354107539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,1,float16,fp8,2047,0.01695466662446658
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,1,float16,float16,4095,0.02109866589307785
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,1,float16,fp8,4095,0.019797333826621372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,1,float16,float16,8191,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,1,float16,fp8,8191,0.024911999702453613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,1,float16,float16,16383,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,1,float16,fp8,16383,0.039247999588648476
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,1,float16,float16,32767,0.10171733299891154
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,1,float16,fp8,32767,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,1,float16,float16,65535,0.18449066082636514
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,1,float16,fp8,65535,0.0993280013402303
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,1,float16,float16,131071,0.35174401601155597
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,1,float16,fp8,131071,0.18228799104690552
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,1,float16,float16,1,0.10274133086204529
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,1,float16,fp8,1,0.07611733178297679
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,1,float16,float16,3,0.10171733299891154
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,1,float16,fp8,3,0.07611733178297679
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,1,float16,fp8,7,0.07611733178297679
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,1,float16,float16,7,0.10274133086204529
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,1,float16,float16,15,0.10103467106819153
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,1,float16,fp8,15,0.07611733178297679
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,1,float16,float16,31,0.10240000486373901
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,1,float16,fp8,31,0.07542933523654938
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,1,float16,float16,63,0.10205866893132527
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,1,float16,fp8,63,0.07441066702206929
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,1,float16,float16,127,0.10172266761461894
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,1,float16,fp8,127,0.07611200213432312
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,1,float16,float16,255,0.10307199756304423
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,1,float16,fp8,255,0.07611733178297679
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,1,float16,float16,511,0.1723680098851522
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,1,float16,fp8,511,0.11708266536394756
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,1,float16,float16,1,0.19097065925598145
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,1,float16,fp8,1,0.14115200440088907
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,1,float16,float16,3,0.1930239995320638
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,1,float16,fp8,3,0.14164800445238748
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,1,float16,float16,7,0.19114132722218832
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,1,float16,fp8,7,0.14148267110188803
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,1,float16,float16,15,0.19114667177200317
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,1,float16,fp8,15,0.14148267110188803
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,1,float16,float16,31,0.1930239995320638
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,1,float16,fp8,31,0.14165332913398743
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,1,float16,float16,63,0.1914880077044169
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,1,float16,fp8,63,0.14148799578348795
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,1,float16,float16,127,0.19131733973821005
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,1,float16,fp8,127,0.13959999879201254
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,1,float16,float16,255,0.19387733936309814
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,1,float16,fp8,255,0.14148799578348795
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,1,float16,float16,1,0.012335999558369318
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,1,float16,float16,3,0.011952000359694162
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,1,float16,fp8,1,0.012698666503032049
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,1,float16,fp8,3,0.012960000584522883
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,1,float16,float16,7,0.01109333336353302
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,1,float16,fp8,7,0.01091733326514562
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,1,float16,float16,15,0.015685333559910457
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,1,float16,fp8,15,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,1,float16,float16,31,0.010847999403874079
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,1,float16,fp8,31,0.012693333129088083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,1,float16,float16,63,0.01109333336353302
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,1,float16,fp8,63,0.012453333785136541
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,1,float16,float16,127,0.01268799975514412
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,1,float16,fp8,127,0.012560000022252401
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,1,float16,float16,255,0.010826667149861654
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,1,float16,fp8,255,0.013354666531085968
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,1,float16,float16,511,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,1,float16,fp8,511,0.012800000607967377
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,1,float16,float16,1023,0.015253332753976187
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,1,float16,fp8,1023,0.014677333335081736
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,1,float16,float16,2047,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,1,float16,fp8,2047,0.01886933296918869
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,1,float16,float16,4095,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,1,float16,fp8,4095,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,1,float16,float16,8191,0.06007466713587443
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,1,float16,fp8,8191,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,1,float16,float16,16383,0.10274133086204529
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,1,float16,fp8,16383,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,1,float16,float16,32767,0.18722132841746011
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,1,float16,fp8,32767,0.10478933652242024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,1,float16,float16,65535,0.3558453321456909
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,1,float16,fp8,65535,0.18858667214711508
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,1,float16,float16,1,0.012560000022252401
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,1,float16,fp8,1,0.01422400027513504
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,1,float16,float16,3,0.012005332857370377
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,1,float16,fp8,3,0.011109333485364914
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,1,float16,float16,7,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,1,float16,fp8,7,0.013962666193644205
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,1,float16,float16,15,0.011034666250149408
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,1,float16,float16,15,0.02868266652027766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,1,float16,fp8,15,0.012362666428089142
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,1,float16,float16,31,0.011877333124478659
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,1,float16,fp8,31,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,1,float16,float16,63,0.012869333227475485
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,1,float16,fp8,63,0.012346666306257248
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,1,float16,float16,127,0.012389333297808966
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,1,float16,fp8,127,0.010863999525705973
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,1,float16,float16,255,0.01381333296497663
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,1,float16,fp8,255,0.01320533330241839
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,1,float16,float16,511,0.012560000022252401
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,1,float16,fp8,511,0.012805332740147909
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,1,float16,float16,1023,0.014906667172908783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,1,float16,fp8,1023,0.014570667097965876
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,1,float16,float16,2047,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,1,float16,fp8,2047,0.02082666630546252
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,1,float16,float16,4095,0.03958933303753535
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,1,float16,fp8,4095,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,1,float16,float16,8191,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,1,float16,fp8,8191,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,1,float16,float16,16383,0.10240000486373901
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,1,float16,fp8,16383,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,1,float16,float16,32767,0.18619734048843384
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,1,float16,fp8,32767,0.10479467113812764
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,1,float16,float16,65535,0.35174401601155597
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,1,float16,float16,1,0.01209066684047381
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,1,float16,fp8,65535,0.18959999084472656
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,1,float16,fp8,1,0.010858666151762009
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,1,float16,float16,3,0.011530666301647821
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,1,float16,fp8,3,0.01098666712641716
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,1,float16,float16,7,0.012624000509579977
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,1,float16,fp8,7,0.010853332777818045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,1,float16,float16,15,0.011530666301647821
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,1,float16,fp8,15,0.010832000523805618
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,1,float16,float16,31,0.011946666985750198
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,1,float16,fp8,31,0.011413333316644033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,1,float16,fp8,63,0.01116266722480456
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,1,float16,float16,63,0.010858666151762009
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,1,float16,float16,127,0.01246400053302447
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,1,float16,fp8,127,0.011264000087976456
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,1,float16,float16,255,0.011152000476916632
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,1,float16,fp8,255,0.011930666863918304
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,1,float16,float16,511,0.013786666095256805
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,1,float16,fp8,511,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,1,float16,float16,1023,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,1,float16,fp8,1023,0.012389333297808966
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,1,float16,float16,2047,0.012682666381200155
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,1,float16,fp8,2047,0.01221866657336553
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,1,float16,float16,4095,0.012709333250919977
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,1,float16,fp8,4095,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,1,float16,float16,8191,0.018863999595244724
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,1,float16,float16,16383,0.021168000996112823
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,1,float16,fp8,8191,0.01870399961868922
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,1,float16,fp8,16383,0.02075200031201045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,1,float16,float16,32767,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,1,float16,fp8,32767,0.022874665757020313
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,1,float16,float16,65535,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,1,float16,fp8,65535,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,1,float16,float16,131071,0.04404266675313314
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,1,float16,fp8,1,0.011941333611806234
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,1,float16,float16,1,0.01051733394463857
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,1,float16,fp8,131071,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,1,float16,float16,3,0.012096000214417776
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,1,float16,fp8,3,0.011247999966144562
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,1,float16,float16,7,0.01051733394463857
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,1,float16,fp8,7,0.012362666428089142
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,1,float16,float16,15,0.012736000120639801
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,1,float16,fp8,15,0.011829332758982977
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,1,float16,fp8,31,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,1,float16,float16,31,0.010522666076819101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,1,float16,float16,63,0.012122667084137598
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,1,float16,fp8,63,0.01184533288081487
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,1,float16,float16,127,0.011503999431928
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,1,float16,fp8,127,0.011034666250149408
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,1,float16,float16,255,0.012181332955757776
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,1,float16,fp8,255,0.011087999989589056
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,1,float16,fp8,511,0.010858666151762009
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,1,float16,float16,1023,0.012970666090647379
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,1,float16,fp8,1023,0.0129120002190272
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,1,float16,float16,2047,0.012458667159080505
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,1,float16,fp8,2047,0.01303999995191892
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,1,float16,float16,4095,0.016672000288963318
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,1,float16,fp8,4095,0.01666133354107539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,1,float16,float16,8191,0.02070933332045873
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,1,float16,fp8,8191,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,1,float16,float16,16383,0.02081599955757459
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,1,float16,fp8,16383,0.020714666694402695
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,1,float16,float16,32767,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,1,float16,fp8,32767,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,1,float16,float16,65535,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,1,float16,fp8,65535,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,1,float16,float16,131071,0.06144000093142191
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,1,float16,fp8,131071,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,1,float16,float16,1,0.01458666721979777
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,1,float16,fp8,1,0.012725333372751871
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,1,float16,float16,3,0.014277332772811254
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,1,float16,fp8,3,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,1,float16,float16,7,0.0145066666106383
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,1,float16,fp8,7,0.012863999853531519
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,1,float16,float16,15,0.0129120002190272
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,1,float16,fp8,15,0.012741333494583765
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,1,float16,float16,31,0.014570667097965876
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,1,float16,fp8,31,0.012736000120639801
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,1,float16,float16,63,0.0145066666106383
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,1,float16,fp8,63,0.014389333625634512
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,1,float16,float16,127,0.014501333236694336
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,1,float16,fp8,127,0.012469333906968435
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,1,float16,float16,255,0.01422400027513504
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,1,float16,fp8,255,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,1,float16,float16,511,0.015978666643301647
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,1,float16,fp8,511,0.014752000570297241
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,1,float16,float16,1023,0.02081599955757459
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,1,float16,fp8,1023,0.018709332992633183
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,1,float16,float16,2047,0.04095999896526337
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,1,float16,fp8,2047,0.02696000039577484
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,1,float16,fp8,4095,0.04164266586303711
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,1,float16,float16,4095,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,1,float16,float16,8191,0.10411199927330017
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,1,float16,fp8,8191,0.06383466720581055
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,1,float16,fp8,16383,0.10717866818110149
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,1,float16,float16,16383,0.18807466824849448
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,1,float16,float16,1,0.010858666151762009
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,1,float16,float16,32767,0.3551573355992635
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,1,float16,fp8,32767,0.1930239995320638
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,1,float16,fp8,1,0.010863999525705973
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,1,float16,float16,3,0.012789333860079447
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,1,float16,fp8,3,0.010911999891201654
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,1,float16,float16,7,0.010858666151762009
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,1,float16,fp8,7,0.01251199965675672
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,1,float16,float16,15,0.011637333780527115
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,1,float16,fp8,15,0.010858666151762009
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,1,float16,float16,31,0.012682666381200155
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,1,float16,fp8,31,0.011823999385039011
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,1,float16,float16,63,0.010863999525705973
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,1,float16,fp8,63,0.012341332932313284
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,1,float16,float16,127,0.010858666151762009
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,1,float16,fp8,127,0.010858666151762009
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,1,float16,float16,255,0.012122667084137598
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,1,float16,fp8,255,0.011029332876205444
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,1,float16,float16,511,0.010687999427318573
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,1,float16,fp8,511,0.012869333227475485
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,1,float16,float16,1023,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,1,float16,fp8,1023,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,1,float16,float16,2047,0.016682667036851246
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,1,float16,fp8,2047,0.01666133354107539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,1,float16,float16,4095,0.01870399961868922
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,1,float16,fp8,4095,0.018677332748969395
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,1,float16,float16,8191,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,1,float16,fp8,8191,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,1,float16,float16,16383,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,1,float16,fp8,16383,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,1,float16,float16,32767,0.04095999896526337
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,1,float16,fp8,32767,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,1,float16,float16,65535,0.06075733403364817
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,1,float16,fp8,65535,0.04164266586303711
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,1,float16,float16,131071,0.10069333513577779
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,1,float16,fp8,131071,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,1,float16,float16,1,0.016656000167131424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,1,float16,fp8,1,0.015087999403476715
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,1,float16,float16,3,0.016666666915019352
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,1,float16,fp8,3,0.014858666807413101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,1,float16,float16,7,0.01682666689157486
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,1,float16,fp8,7,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,1,float16,float16,15,0.016901332885026932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,1,float16,fp8,15,0.016634666671355564
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,1,float16,float16,31,0.01672533278663953
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,1,float16,float16,63,0.016800000021855038
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,1,float16,fp8,63,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,1,float16,fp8,31,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,1,float16,float16,127,0.016805333395799
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,1,float16,fp8,127,0.014848000059525171
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,1,float16,float16,255,0.01672533278663953
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,1,float16,fp8,255,0.015082667271296183
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,1,float16,float16,511,0.02219199885924657
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,1,float16,fp8,511,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,1,float16,fp8,1023,0.024847999215126038
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,1,float16,float16,1023,0.03549866626660029
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,1,float16,float16,2047,0.06178666651248932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,1,float16,fp8,2047,0.04266666869322459
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,1,float16,float16,4095,0.10478400190671285
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,1,float16,fp8,4095,0.06417066852251689
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,1,float16,float16,8191,0.18858667214711508
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,1,float16,fp8,8191,0.10820266604423523
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,1,float16,float16,16383,0.358570655186971
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,1,float16,float16,1,0.02275199939807256
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,1,float16,fp8,16383,0.19337066014607748
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,1,float16,fp8,1,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,1,float16,float16,3,0.02276266614596049
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,1,float16,fp8,3,0.018613333503405254
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,1,float16,float16,7,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,1,float16,fp8,7,0.019706666469573975
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,1,float16,float16,15,0.02184533327817917
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,1,float16,fp8,15,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,1,float16,float16,31,0.02276266614596049
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,1,float16,fp8,31,0.01870399961868922
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,1,float16,float16,63,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,1,float16,fp8,63,0.01871466636657715
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,1,float16,float16,127,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,1,float16,fp8,127,0.018709332992633183
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,1,float16,float16,255,0.02276266614596049
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,1,float16,fp8,255,0.01871466636657715
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,1,float16,float16,511,0.03345066557327906
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,1,float16,fp8,511,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,1,float16,float16,1023,0.05358933409055074
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,1,float16,fp8,1023,0.03515200068553289
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,1,float16,float16,2047,0.09830933809280396
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,1,float16,fp8,2047,0.06144000093142191
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,1,float16,float16,4095,0.17527467012405396
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,1,float16,fp8,4095,0.09864532947540283
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,1,float16,float16,1,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,1,float16,fp8,1,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,1,float16,float16,3,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,1,float16,fp8,3,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,1,float16,float16,7,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,1,float16,fp8,7,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,1,float16,float16,15,0.03311466674009959
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,1,float16,fp8,15,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,1,float16,float16,31,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,1,float16,fp8,31,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,1,float16,float16,63,0.03310399999221166
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,1,float16,fp8,63,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,1,float16,float16,127,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,1,float16,fp8,127,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,1,float16,float16,255,0.03616533428430557
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,1,float16,fp8,255,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,1,float16,float16,511,0.05495466788609823
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,1,float16,fp8,511,0.039936001102129616
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,1,float16,float16,1023,0.09284266829490662
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,1,float16,fp8,1023,0.0576853354771932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,1,float16,float16,2047,0.176639993985494
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,1,float16,fp8,2047,0.10274133086204529
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,1,float16,float16,1,0.010341333225369453
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,1,float16,fp8,1,0.012485332787036896
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,1,float16,float16,3,0.012128000458081564
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,1,float16,fp8,3,0.011605333536863327
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,1,float16,float16,7,0.011264000087976456
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,1,float16,fp8,7,0.011957333733638128
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,1,float16,float16,15,0.010687999427318573
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,1,float16,fp8,15,0.012117333710193634
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,1,float16,float16,31,0.011226666470368704
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,1,float16,fp8,31,0.010869332899649939
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,1,float16,float16,63,0.012586666891972223
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,1,float16,fp8,63,0.011034666250149408
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,1,float16,float16,127,0.010853332777818045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,1,float16,fp8,127,0.011157333850860596
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,1,float16,float16,255,0.010858666151762009
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,1,float16,float16,511,0.01249066616098086
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,1,float16,fp8,255,0.010869332899649939
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,1,float16,fp8,511,0.011717333147923151
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,1,float16,float16,1023,0.013034666577974955
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,1,float16,fp8,1023,0.012554666648308435
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,1,float16,float16,2047,0.014954666296641031
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,1,float16,fp8,2047,0.01647466669480006
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,1,float16,float16,4095,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,1,float16,fp8,4095,0.016800000021855038
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,1,float16,float16,8191,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,1,float16,fp8,8191,0.019098666807015736
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,1,float16,float16,16383,0.038912000755469
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,1,float16,fp8,16383,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,1,float16,float16,32767,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,1,float16,fp8,32767,0.03993066648642222
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,1,float16,float16,65535,0.09966933727264404
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,1,float16,fp8,65535,0.059392000238100685
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,1,float16,float16,131071,0.18345065911610922
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,1,float16,fp8,131071,0.10102933645248413
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,1,float16,float16,1,0.05598400036493937
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,1,float16,fp8,1,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,1,float16,float16,3,0.05563733478387197
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,1,float16,fp8,3,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,1,float16,float16,7,0.055642664432525635
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,1,float16,fp8,7,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,1,float16,float16,15,0.057002668579419456
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,1,float16,fp8,15,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,1,float16,float16,31,0.05563733478387197
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,1,float16,float16,63,0.05562133093674978
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,1,float16,fp8,63,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,1,float16,float16,127,0.0576853354771932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,1,float16,fp8,127,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,1,float16,float16,255,0.0576853354771932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,1,float16,fp8,255,0.04540266593297323
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,1,float16,float16,511,0.09351467092831929
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,1,float16,fp8,511,0.06587733328342438
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,1,float16,float16,1023,0.1687893271446228
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,1,float16,fp8,1023,0.1013759970664978
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,1,float16,float16,1,0.10273067156473796
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,1,float16,fp8,1,0.07441066702206929
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,1,float16,float16,3,0.10274133086204529
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,1,float16,fp8,3,0.0744053324063619
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,1,float16,float16,7,0.10274133086204529
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,1,float16,fp8,7,0.0747519979874293
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,1,float16,float16,15,0.10274133086204529
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,1,float16,fp8,15,0.07578133543332417
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,1,float16,float16,31,0.10274133086204529
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,1,float16,fp8,31,0.07406400144100189
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,1,float16,float16,63,0.10273599624633789
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,1,float16,fp8,63,0.07577600081761678
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,1,float16,float16,127,0.10069333513577779
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,1,float16,fp8,127,0.07611733178297679
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,1,float16,float16,255,0.10240000486373901
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,1,float16,fp8,255,0.07816533247629802
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,1,float16,float16,511,0.17220799128214517
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,1,float16,float16,1,0.01051733394463857
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,1,float16,fp8,511,0.11707733074824016
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,1,float16,fp8,1,0.011535999675591787
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,1,float16,float16,3,0.012351999680201212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,1,float16,fp8,3,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,1,float16,float16,7,0.012266666938861212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,1,float16,fp8,7,0.011546666423479715
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,1,float16,float16,15,0.011605333536863327
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,1,float16,fp8,15,0.011685332904259363
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,1,float16,fp8,31,0.01221866657336553
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,1,float16,float16,31,0.012703999876976013
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,1,float16,float16,63,0.010682666053374609
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,1,float16,fp8,63,0.010832000523805618
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,1,float16,float16,127,0.010687999427318573
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,1,float16,fp8,127,0.011765333513418833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,1,float16,float16,255,0.01051733394463857
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,1,float16,fp8,255,0.012229333321253458
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,1,float16,float16,511,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,1,float16,fp8,511,0.014208000153303146
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,1,float16,float16,1023,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,1,float16,fp8,1023,0.014576000471909841
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,1,float16,float16,2047,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,1,float16,fp8,2047,0.016965333372354507
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,1,float16,float16,4095,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,1,float16,fp8,4095,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,1,float16,float16,8191,0.039247999588648476
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,1,float16,fp8,8191,0.024911999702453613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,1,float16,float16,16383,0.06075733403364817
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,1,float16,fp8,16383,0.039247999588648476
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,1,float16,float16,32767,0.10103467106819153
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,1,float16,fp8,32767,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,1,float16,float16,65535,0.18312533696492514
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,1,float16,fp8,65535,0.10069333513577779
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,1,float16,float16,131071,0.3490133285522461
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,1,float16,fp8,131071,0.18397333224614462
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,1,float16,float16,1,0.04949333270390829
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,1,float16,fp8,1,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,1,float16,float16,3,0.051541333397229515
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,1,float16,fp8,3,0.043354665239652
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,1,float16,float16,7,0.051541333397229515
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,1,float16,fp8,7,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,1,float16,float16,15,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,1,float16,fp8,15,0.05563733478387197
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,1,float16,float16,31,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,1,float16,fp8,31,0.05392000079154968
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,1,float16,float16,63,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,1,float16,fp8,63,0.05358933409055074
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,1,float16,float16,127,0.07406933108965556
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,1,float16,fp8,127,0.06383466720581055
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,1,float16,float16,255,0.1088853379090627
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,1,float16,fp8,255,0.09659733374913533
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,1,float16,float16,511,0.17834667364756265
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,1,float16,fp8,511,0.16025066375732422
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,1,float16,float16,1023,0.31624533732732135
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,1,float16,fp8,1023,0.2903093298276265
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,1,float16,float16,2047,0.5939199924468994
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,1,float16,fp8,2047,0.5509119828542074
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,1,float16,float16,4095,1.1559253533681233
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,1,float16,fp8,4095,1.0719520250956218
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,1,float16,float16,1,0.04949333270390829
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,1,float16,fp8,1,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,1,float16,float16,3,0.05153599878152212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,1,float16,fp8,3,0.04334400097529093
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,1,float16,float16,7,0.051541333397229515
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,1,float16,fp8,7,0.043354665239652
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,1,float16,float16,15,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,1,float16,fp8,15,0.05358933409055074
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,1,float16,float16,31,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,1,float16,fp8,31,0.05563733478387197
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,1,float16,float16,511,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,1,float16,float16,63,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,1,float16,fp8,63,0.05358933409055074
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,1,float16,float16,127,0.07407466570536296
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,1,float16,fp8,127,0.06382933259010315
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,1,float16,float16,255,0.1088853379090627
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,1,float16,fp8,255,0.09659733374913533
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,1,float16,float16,511,0.17851734161376953
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,1,float16,fp8,511,0.16196266810099283
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,1,float16,float16,1023,0.31590400139490765
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,1,float16,fp8,1023,0.2913279930750529
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,1,float16,float16,2047,0.5990399916966757
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,1,float16,fp8,2047,0.547162652015686
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,1,float16,float16,4095,1.1939786275227864
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,1,float16,fp8,4095,1.0746986865997314
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,1,float16,float16,1,0.049498667319615684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,1,float16,fp8,1,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,1,float16,float16,3,0.051541333397229515
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,1,float16,fp8,3,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,1,float16,float16,7,0.051541333397229515
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,1,float16,fp8,7,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,1,float16,float16,15,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,1,float16,fp8,15,0.05358933409055074
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,1,float16,float16,31,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,1,float16,fp8,31,0.053930665055910744
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,1,float16,float16,63,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,1,float16,fp8,63,0.053599998354911804
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,1,float16,float16,127,0.07406933108965556
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,1,float16,fp8,127,0.06382933259010315
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,1,float16,float16,255,0.11024533708890279
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,1,float16,fp8,255,0.09659733374913533
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,1,float16,float16,511,0.17834667364756265
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,1,float16,fp8,511,0.16025599837303162
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,1,float16,float16,1023,0.31590400139490765
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,1,float16,fp8,1023,0.29098133246103924
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,1,float16,float16,2047,0.6312959988911947
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,1,float16,fp8,2047,0.5488640069961548
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,1,float16,float16,4095,1.2798293431599934
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,1,float16,float16,1,0.01594666639963786
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,1,float16,fp8,4095,1.0777546564737956
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,1,float16,fp8,1,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,1,float16,float16,3,0.014959999670584997
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,1,float16,fp8,3,0.014741333822409311
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,1,float16,float16,7,0.014949332922697067
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,1,float16,fp8,7,0.014666666587193808
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,1,float16,float16,15,0.015072000523408255
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,1,float16,fp8,15,0.014789332946141561
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,1,float16,float16,31,0.016613333175579708
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,1,float16,fp8,31,0.014736000448465347
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,1,float16,float16,63,0.014677333335081736
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,1,float16,fp8,63,0.014565333724021912
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,1,float16,float16,127,0.014906667172908783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,1,float16,fp8,127,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,1,float16,float16,255,0.016613333175579708
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,1,float16,fp8,255,0.014725333700577417
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,1,float16,float16,511,0.01666133354107539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,1,float16,fp8,511,0.016607999801635742
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,1,float16,float16,1023,0.02242133269707362
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,1,float16,fp8,1023,0.01904533306757609
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,1,float16,float16,2047,0.04095999896526337
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,1,float16,fp8,2047,0.029018667836983997
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,1,float16,fp8,4095,0.04471466441949209
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,1,float16,float16,4095,0.062122667829195656
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,1,float16,float16,1,0.011482667177915573
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,1,float16,fp8,1,0.010410666465759277
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,1,float16,float16,3,0.010399999717871347
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,1,float16,fp8,3,0.011834666132926941
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,1,float16,float16,7,0.010746666540702185
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,1,float16,fp8,7,0.009818666925032934
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,1,float16,float16,15,0.011541333049535751
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,1,float16,fp8,15,0.010527999450763067
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,1,float16,float16,31,0.010421333213647207
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,1,float16,fp8,31,0.01251199965675672
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,1,float16,float16,63,0.011359999577204386
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,1,float16,fp8,63,0.01091733326514562
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,1,float16,float16,127,0.01221866657336553
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,1,float16,fp8,127,0.010992000500361124
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,1,float16,float16,255,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,1,float16,fp8,255,0.02276266614596049
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,1,float16,float16,511,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,1,float16,fp8,511,0.024911999702453613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,1,float16,float16,1023,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,1,float16,fp8,1023,0.031066666046778362
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,1,float16,float16,2047,0.04539200166861216
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,1,float16,fp8,2047,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,1,float16,float16,4095,0.07202133536338806
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,1,float16,fp8,4095,0.06587733328342438
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,1,float16,float16,8191,0.12390399972597758
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,1,float16,fp8,8191,0.11127466956774394
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,1,float16,float16,16383,0.22562134265899658
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,1,float16,fp8,16383,0.2039466698964437
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,1,float16,float16,1,0.011322667201360067
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,1,float16,fp8,1,0.011493333925803503
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,1,float16,float16,3,0.01198400060335795
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,1,float16,fp8,3,0.010469333579142889
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,1,float16,float16,7,0.011717333147923151
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,1,float16,fp8,7,0.011482667177915573
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,1,float16,float16,15,0.010362666721145311
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,1,float16,fp8,15,0.011461333682139715
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,1,float16,float16,31,0.012773333738247553
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,1,float16,fp8,31,0.010469333579142889
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,1,float16,float16,63,0.011312000453472137
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,1,float16,float16,127,0.011434666812419891
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,1,float16,fp8,63,0.011898666620254517
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,1,float16,fp8,127,0.011978667229413986
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,1,float16,float16,255,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,1,float16,fp8,255,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,1,float16,float16,511,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,1,float16,fp8,511,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,1,float16,float16,1023,0.03310399999221166
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,1,float16,fp8,1023,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,1,float16,float16,2047,0.04539733131726583
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,1,float16,fp8,2047,0.04334400097529093
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,1,float16,float16,4095,0.07202133536338806
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,1,float16,fp8,4095,0.06587733328342438
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,1,float16,float16,8191,0.12390399972597758
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,1,float16,fp8,8191,0.11126933495203654
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,1,float16,float16,16383,0.22562134265899658
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,1,float16,fp8,16383,0.2032639980316162
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,1,float16,float16,1,0.009989333028594652
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,1,float16,fp8,1,0.01090666651725769
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,1,float16,float16,3,0.011706666400035223
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,1,float16,fp8,3,0.011600000162919363
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,1,float16,float16,7,0.011509332805871964
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,1,float16,fp8,7,0.011594666788975397
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,1,float16,float16,15,0.011418666690587997
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,1,float16,fp8,15,0.011663999408483505
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,1,float16,float16,31,0.012378666549921036
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,1,float16,fp8,31,0.01198400060335795
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,1,float16,float16,63,0.011407999942700068
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,1,float16,fp8,63,0.012362666428089142
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,1,float16,float16,127,0.011653333902359009
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,1,float16,fp8,127,0.010650667051474253
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,1,float16,float16,255,0.022757334013779957
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,1,float16,fp8,255,0.02082666630546252
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,1,float16,float16,511,0.025941332181294758
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,1,float16,fp8,511,0.025146665672461193
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,1,float16,float16,1023,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,1,float16,fp8,1023,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,1,float16,float16,2047,0.047450666626294456
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,1,float16,fp8,2047,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,1,float16,float16,4095,0.07201600074768066
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,1,float16,fp8,4095,0.06587733328342438
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,1,float16,float16,8191,0.12322133779525757
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,1,float16,fp8,8191,0.11128000418345134
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,1,float16,float16,1,0.012634667257467905
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,1,float16,fp8,16383,0.2032639980316162
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,1,float16,float16,16383,0.22766933838526407
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,1,float16,fp8,1,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,1,float16,float16,3,0.010853332777818045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,1,float16,fp8,3,0.011850666254758835
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,1,float16,float16,7,0.01116266722480456
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,1,float16,fp8,7,0.012053333222866058
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,1,float16,float16,15,0.012175999581813812
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,1,float16,fp8,15,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,1,float16,float16,31,0.01198400060335795
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,1,float16,fp8,31,0.01211200033624967
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,1,float16,float16,63,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,1,float16,fp8,63,0.012240000069141388
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,1,float16,float16,127,0.012181332955757776
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,1,float16,fp8,127,0.011168000598748526
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,1,float16,float16,255,0.012165332833925882
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,1,float16,fp8,255,0.012800000607967377
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,1,float16,float16,511,0.012533333152532578
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,1,float16,fp8,511,0.012378666549921036
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,1,float16,float16,1023,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,1,float16,fp8,1023,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,1,float16,float16,2047,0.012736000120639801
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,1,float16,fp8,2047,0.01268799975514412
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,1,float16,float16,4095,0.017664000391960144
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,1,float16,fp8,4095,0.018170667191346485
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,1,float16,float16,8191,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,1,float16,fp8,8191,0.020479999482631683
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,1,float16,float16,16383,0.022874665757020313
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,1,float16,fp8,16383,0.021061333517233532
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,1,float16,float16,1,0.014778666198253632
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,1,float16,fp8,1,0.011722666521867117
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,1,float16,float16,3,0.014762666076421738
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,1,float16,fp8,3,0.01044800008336703
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,1,float16,float16,7,0.014602666099866232
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,1,float16,fp8,7,0.011690666278203329
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,1,float16,float16,15,0.014565333724021912
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,1,float16,fp8,15,0.012069333344697952
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,1,float16,float16,31,0.016672000288963318
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,1,float16,fp8,31,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,1,float16,float16,63,0.016672000288963318
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,1,float16,fp8,63,0.012693333129088083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,1,float16,float16,127,0.016810666769742966
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,1,float16,fp8,127,0.012741333494583765
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,1,float16,float16,255,0.0186666672428449
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,1,float16,fp8,255,0.014767999450365702
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,1,float16,float16,511,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,1,float16,float16,1023,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,1,float16,fp8,511,0.020703999946514767
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,1,float16,fp8,1023,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,1,float16,float16,2047,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,1,float16,fp8,2047,0.04949333270390829
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,1,float16,float16,4095,0.1088853379090627
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,1,float16,fp8,4095,0.08840533097585042
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,1,float16,float16,8191,0.20633065700531006
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,1,float16,fp8,8191,0.16828266779581705
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,1,float16,float16,16383,0.3959519863128662
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,1,float16,fp8,16383,0.32683199644088745
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,1,float16,float16,1,0.01461333284775416
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,1,float16,fp8,1,0.010650667051474253
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,1,float16,float16,3,0.014405333747466406
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,1,float16,float16,7,0.014794666320085526
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,1,float16,fp8,3,0.011440000186363855
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,1,float16,fp8,7,0.012074666718641916
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,1,float16,float16,15,0.014917333920796713
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,1,float16,fp8,15,0.01157333329319954
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,1,float16,float16,31,0.016607999801635742
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,1,float16,fp8,31,0.01268799975514412
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,1,float16,float16,63,0.016554666062196095
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,1,float16,fp8,63,0.014570667097965876
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,1,float16,float16,127,0.016810666769742966
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,1,float16,fp8,127,0.012741333494583765
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,1,float16,float16,255,0.0186666672428449
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,1,float16,fp8,255,0.014762666076421738
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,1,float16,float16,511,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,1,float16,fp8,511,0.02070933332045873
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,1,float16,float16,1023,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,1,float16,fp8,1023,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,1,float16,float16,2047,0.06177600224812826
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,1,float16,fp8,2047,0.04949333270390829
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,1,float16,float16,4095,0.1088853379090627
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,1,float16,fp8,31,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,1,float16,fp8,4095,0.08942932883898418
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,1,float16,float16,8191,0.20514132579167685
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,1,float16,fp8,8191,0.1667520006497701
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,1,float16,float16,16383,0.39628799756368
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,1,float16,fp8,16383,0.32443734010060626
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,1,float16,float16,1,0.014565333724021912
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,1,float16,fp8,1,0.012181332955757776
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,1,float16,float16,3,0.014602666099866232
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,1,float16,fp8,3,0.011999999483426413
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,1,float16,float16,7,0.0144213338692983
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,1,float16,fp8,7,0.011973333855470022
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,1,float16,float16,15,0.014837333311637243
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,1,float16,fp8,15,0.011946666985750198
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,1,float16,float16,31,0.016688000410795212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,1,float16,fp8,31,0.012741333494583765
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,1,float16,float16,63,0.01666133354107539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,1,float16,fp8,63,0.01268799975514412
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,1,float16,float16,127,0.016629333297411602
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,1,float16,fp8,127,0.013141332815090815
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,1,float16,float16,255,0.018709332992633183
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,1,float16,fp8,255,0.014565333724021912
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,1,float16,float16,511,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,1,float16,fp8,511,0.020080000162124634
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,1,float16,fp8,1023,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,1,float16,float16,2047,0.06178666651248932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,1,float16,float16,1023,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,1,float16,fp8,2047,0.04949333270390829
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,1,float16,fp8,4095,0.08841066559155782
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,1,float16,float16,4095,0.1109333336353302
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,1,float16,float16,8191,0.20531733830769858
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,1,float16,fp8,8191,0.16742400328318277
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,1,float16,float16,16383,0.3973120053609212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,1,float16,fp8,16383,0.32547199726104736
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,1,float16,float16,1,0.010682666053374609
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,1,float16,fp8,1,0.011786667009194693
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,1,float16,float16,3,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,1,float16,fp8,3,0.01210133358836174
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,1,float16,float16,7,0.010693332801262537
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,1,float16,fp8,7,0.012432000289360682
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,1,float16,float16,15,0.012357333054145178
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,1,float16,float16,31,0.011424000064531961
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,1,float16,fp8,31,0.011717333147923151
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,1,float16,float16,63,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,1,float16,fp8,63,0.010853332777818045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,1,float16,float16,127,0.011850666254758835
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,1,float16,fp8,127,0.011498666057984034
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,1,float16,float16,255,0.011018666128317514
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,1,float16,fp8,255,0.012229333321253458
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,1,float16,float16,511,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,1,float16,fp8,511,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,1,float16,float16,1023,0.012229333321253458
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,1,float16,fp8,1023,0.012863999853531519
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,1,float16,float16,2047,0.016656000167131424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,1,float16,fp8,2047,0.016666666915019352
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,1,float16,float16,4095,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,1,float16,fp8,4095,0.018709332992633183
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,1,float16,float16,8191,0.02075200031201045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,1,float16,fp8,8191,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,1,float16,float16,16383,0.02492266645034154
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,1,float16,fp8,16383,0.022874665757020313
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,1,float16,float16,1,0.09454933802286784
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,1,float16,float16,3,0.09659733374913533
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,1,float16,fp8,1,0.07611733178297679
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,1,float16,fp8,3,0.07816533247629802
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,1,float16,float16,7,0.09864532947540283
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,1,float16,fp8,7,0.08226133386294048
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,1,float16,float16,15,0.11707733074824016
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,1,float16,fp8,15,0.10070400436719258
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,1,float16,float16,31,0.11707733074824016
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,1,float16,fp8,31,0.10274133086204529
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,1,float16,float16,63,0.11741866668065389
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,1,float16,float16,127,0.13959999879201254
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,1,float16,fp8,63,0.1013759970664978
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,1,float16,fp8,127,0.12117333213488261
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,1,float16,float16,255,0.20923733711242676
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,1,float16,fp8,255,0.18466132879257202
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,1,float16,fp8,511,0.31010133028030396
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,1,float16,float16,511,0.3473066488901774
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,1,float16,float16,1023,0.621562679608663
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,1,float16,fp8,1023,0.5672959884007772
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,1,float16,float16,2047,1.1712853113810222
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,1,float16,fp8,2047,1.0821973482767742
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,1,float16,float16,1,0.09454933802286784
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,1,float16,fp8,1,0.07611733178297679
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,1,float16,float16,3,0.09659733374913533
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,1,float16,fp8,3,0.07816533247629802
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,1,float16,float16,7,0.09864532947540283
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,1,float16,fp8,7,0.08191999793052673
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,1,float16,float16,15,0.11707733074824016
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,1,float16,fp8,15,0.10205866893132527
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,1,float16,float16,31,0.11707733074824016
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,1,float16,fp8,31,0.10069866975148518
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,1,float16,float16,63,0.11707733074824016
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,1,float16,fp8,63,0.10274133086204529
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,1,float16,float16,127,0.1397760013739268
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,1,float16,fp8,127,0.12117333213488261
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,1,float16,float16,255,0.20923733711242676
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,1,float16,fp8,255,0.18466132879257202
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,1,float16,float16,511,0.3466293414433797
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,1,float16,fp8,511,0.3114666740099589
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,1,float16,float16,1023,0.6212266683578491
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,1,float16,fp8,1023,0.5713920195897421
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,1,float16,float16,2047,1.2137813568115234
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,1,float16,fp8,2047,1.0808320045471191
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,1,float16,float16,1,0.09454933802286784
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,1,float16,fp8,1,0.07611733178297679
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,1,float16,float16,3,0.09659733374913533
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,1,float16,fp8,3,0.07816533247629802
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,1,float16,float16,7,0.0986400047938029
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,1,float16,fp8,7,0.08226133386294048
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,1,float16,float16,15,0.11707733074824016
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,1,float16,fp8,15,0.10274133086204529
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,1,float16,float16,31,0.11707733074824016
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,1,float16,fp8,31,0.10068266590436299
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,1,float16,float16,63,0.11878400047620137
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,1,float16,fp8,63,0.10274133086204529
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,1,float16,float16,127,0.14046399792035422
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,1,float16,float16,255,0.2092426617940267
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,1,float16,fp8,127,0.12117333213488261
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,1,float16,fp8,255,0.18449066082636514
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,1,float16,float16,511,0.3466240167617798
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,1,float16,fp8,511,0.3097599943478902
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,1,float16,float16,1023,0.6473386685053507
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,1,float16,fp8,1023,0.5672959884007772
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,1,float16,float16,2047,1.2716373602549236
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,1,float16,fp8,2047,1.087999979654948
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,1,float16,float16,1,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,1,float16,fp8,1,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,1,float16,float16,3,0.02081599955757459
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,1,float16,fp8,3,0.0186666672428449
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,1,float16,fp8,7,0.01871466636657715
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,1,float16,float16,7,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,1,float16,float16,15,0.020714666694402695
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,1,float16,fp8,15,0.018698666244745255
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,1,float16,float16,31,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,1,float16,fp8,31,0.018661333868900936
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,1,float16,float16,63,0.01904533306757609
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,1,float16,fp8,63,0.018709332992633183
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,1,float16,float16,127,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,1,float16,fp8,127,0.01803733284274737
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,1,float16,float16,255,0.020479999482631683
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,1,float16,fp8,255,0.018661333868900936
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,1,float16,float16,511,0.024522667129834492
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,1,float16,fp8,511,0.02252800017595291
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,1,float16,float16,1023,0.03788800040880839
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,1,float16,fp8,1023,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,1,float16,float16,2047,0.06382933259010315
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,1,float16,fp8,2047,0.04539733131726583
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,1,float16,float16,1,0.01461333284775416
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,1,float16,fp8,1,0.014938666174809137
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,1,float16,float16,3,0.014565333724021912
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,1,float16,fp8,3,0.016666666915019352
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,1,float16,float16,7,0.014511999984582266
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,1,float16,fp8,7,0.016623999923467636
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,1,float16,float16,15,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,1,float16,fp8,15,0.01657066618402799
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,1,float16,float16,31,0.014752000570297241
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,1,float16,fp8,31,0.020773333807786305
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,1,float16,float16,63,0.014602666099866232
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,1,float16,fp8,63,0.02070933332045873
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,1,float16,float16,127,0.016997333616018295
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,1,float16,fp8,127,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,1,float16,float16,255,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,1,float16,fp8,255,0.023557332654794056
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,1,float16,fp8,511,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,1,float16,float16,511,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,1,float16,float16,1023,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,1,float16,fp8,1023,0.05358933409055074
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,1,float16,float16,2047,0.10683733224868774
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,1,float16,fp8,2047,0.09045333663622539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,1,float16,float16,4095,0.20121600230534872
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,1,float16,fp8,4095,0.16708266735076904
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,1,float16,float16,8191,0.39168532689412433
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,1,float16,fp8,8191,0.3199999928474426
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,1,float16,float16,16383,0.7878080209096273
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,1,float16,fp8,16383,0.6256639957427979
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,1,float16,float16,1,0.01463466634353002
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,1,float16,fp8,1,0.014869333555301031
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,1,float16,float16,3,0.014778666198253632
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,1,float16,fp8,3,0.01647466669480006
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,1,float16,float16,7,0.014853333433469137
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,1,float16,float16,15,0.01463466634353002
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,1,float16,fp8,7,0.016677333662907284
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,1,float16,fp8,15,0.01666133354107539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,1,float16,float16,31,0.014677333335081736
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,1,float16,fp8,31,0.02075200031201045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,1,float16,float16,63,0.014853333433469137
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,1,float16,fp8,63,0.02081599955757459
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,1,float16,float16,127,0.016783999900023144
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,1,float16,fp8,127,0.02081599955757459
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,1,float16,float16,255,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,1,float16,fp8,255,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,1,float16,float16,511,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,1,float16,fp8,511,0.03310399999221166
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,1,float16,float16,1023,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,1,float16,fp8,1023,0.051541333397229515
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,1,float16,float16,2047,0.10684266686439514
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,1,float16,fp8,2047,0.09045333663622539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,1,float16,float16,4095,0.20224533478418985
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,1,float16,fp8,4095,0.1662399967511495
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,1,float16,float16,8191,0.3969759941101074
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,1,float16,fp8,8191,0.31965865691502887
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,1,float16,float16,16383,0.8029867013295492
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,1,float16,fp8,16383,0.6253226598103842
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,1,float16,fp8,1,0.014890667051076889
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,1,float16,float16,1,0.014501333236694336
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,1,float16,float16,3,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,1,float16,fp8,3,0.01661866654952367
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,1,float16,float16,7,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,1,float16,fp8,7,0.01666133354107539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,1,float16,float16,15,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,1,float16,fp8,15,0.01666133354107539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,1,float16,float16,31,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,1,float16,fp8,31,0.02070933332045873
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,1,float16,float16,63,0.014746667196353277
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,1,float16,fp8,63,0.02075200031201045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,1,float16,float16,127,0.01695466662446658
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,1,float16,fp8,127,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,1,float16,float16,255,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,1,float16,fp8,255,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,1,float16,float16,511,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,1,float16,fp8,511,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,1,float16,float16,1023,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,1,float16,fp8,1023,0.05358933409055074
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,1,float16,float16,2047,0.10683733224868774
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,1,float16,fp8,2047,0.09045867125193278
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,1,float16,float16,4095,0.20121600230534872
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,1,float16,fp8,4095,0.16810667514801025
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,1,float16,float16,8191,0.3969706694285075
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,1,float16,fp8,8191,0.3199999928474426
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,1,float16,float16,16383,0.8115253448486328
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,1,float16,fp8,16383,0.6263413429260254
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,1,float16,float16,1,0.010863999525705973
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,1,float16,fp8,1,0.010858666151762009
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,1,float16,float16,3,0.012693333129088083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,1,float16,float16,7,0.012831999609867731
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,1,float16,fp8,3,0.010853332777818045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,1,float16,fp8,7,0.012015999605258306
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,1,float16,float16,15,0.011434666812419891
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,1,float16,fp8,15,0.010522666076819101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,1,float16,float16,31,0.01098666712641716
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,1,float16,fp8,31,0.011877333124478659
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,1,float16,float16,63,0.01109333336353302
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,1,float16,fp8,63,0.011823999385039011
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,1,float16,float16,127,0.01211200033624967
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,1,float16,fp8,127,0.012367999802033106
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,1,float16,float16,255,0.010992000500361124
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,1,float16,fp8,255,0.012319999436537424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,1,float16,float16,511,0.012383999923865
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,1,float16,fp8,511,0.012863999853531519
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,1,float16,float16,1023,0.013077333569526672
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,1,float16,fp8,1023,0.012560000022252401
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,1,float16,float16,2047,0.01695466662446658
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,1,float16,fp8,2047,0.016554666062196095
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,1,float16,fp8,4095,0.018661333868900936
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,1,float16,float16,4095,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,1,float16,float16,8191,0.022800001005331676
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,1,float16,fp8,8191,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,1,float16,float16,16383,0.03822399924198786
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,1,float16,fp8,16383,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,1,float16,float16,1,0.18278400103251138
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,1,float16,fp8,1,0.14421332875887552
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,1,float16,float16,3,0.18824533621470133
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,1,float16,fp8,3,0.14967466394106546
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,1,float16,float16,7,0.19063466787338257
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,1,float16,float16,15,0.22562134265899658
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,1,float16,fp8,7,0.15598932902018228
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,1,float16,fp8,15,0.1967786749204
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,1,float16,float16,31,0.22766933838526407
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,1,float16,fp8,31,0.1969546675682068
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,1,float16,float16,63,0.22937599817911783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,1,float16,fp8,63,0.1971199909845988
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,1,float16,float16,127,0.2728959918022156
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,1,float16,fp8,127,0.23381332556406656
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,1,float16,float16,255,0.4082346757253011
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,1,float16,fp8,255,0.36061867078145343
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,1,float16,float16,511,0.6833493709564209
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,1,float16,fp8,511,0.6123520135879517
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,1,float16,float16,1023,1.2311893304189045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,1,float16,fp8,1023,1.1245226860046387
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,1,float16,float16,1,0.18278400103251138
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,1,float16,fp8,1,0.14387200276056925
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,1,float16,float16,3,0.18756266434987387
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,1,float16,fp8,3,0.14900267124176025
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,1,float16,float16,7,0.19063466787338257
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,1,float16,fp8,7,0.15598932902018228
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,1,float16,float16,15,0.22664533058802286
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,1,float16,fp8,15,0.1971199909845988
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,1,float16,float16,31,0.22937599817911783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,1,float16,fp8,31,0.1969546675682068
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,1,float16,float16,63,0.22971733411153158
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,1,float16,fp8,63,0.19694934288660684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,1,float16,float16,127,0.2728959918022156
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,1,float16,fp8,127,0.23381332556406656
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,1,float16,float16,255,0.4082346757253011
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,1,float16,fp8,255,0.3609600067138672
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,1,float16,float16,511,0.6843732992808024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,1,float16,fp8,511,0.6133866707483927
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,1,float16,float16,1023,1.2641279697418213
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,1,float16,fp8,1023,1.1252000331878662
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,1,float16,float16,1,0.18278400103251138
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,1,float16,fp8,1,0.1437013347943624
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,1,float16,float16,3,0.1868799924850464
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,1,float16,fp8,3,0.14847999811172485
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,1,float16,float16,7,0.19063466787338257
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,1,float16,fp8,7,0.15615999698638916
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,1,float16,float16,15,0.22562134265899658
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,1,float16,fp8,15,0.196943998336792
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,1,float16,float16,31,0.22971733411153158
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,1,float16,fp8,31,0.1971199909845988
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,1,float16,float16,63,0.22971733411153158
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,1,float16,fp8,63,0.19746132691701254
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,1,float16,fp8,127,0.23381332556406656
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,1,float16,float16,127,0.2728959918022156
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,1,float16,float16,255,0.40960001945495605
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,1,float16,fp8,255,0.3609600067138672
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,1,float16,float16,511,0.7174826463063558
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,1,float16,fp8,511,0.6123520135879517
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,1,float16,float16,1023,1.3199360370635986
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,1,float16,fp8,1023,1.1333973407745361
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,1,float16,float16,1,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,1,float16,fp8,1,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,1,float16,float16,3,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,1,float16,fp8,3,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,1,float16,float16,7,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,1,float16,fp8,7,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,1,float16,float16,15,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,1,float16,fp8,15,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,1,float16,float16,31,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,1,float16,fp8,31,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,1,float16,float16,63,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,1,float16,fp8,63,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,1,float16,float16,127,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,1,float16,fp8,127,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,1,float16,float16,255,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,1,float16,fp8,255,0.02492800106604894
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,1,float16,float16,511,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,1,float16,fp8,511,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,1,float16,float16,1023,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,1,float16,fp8,1023,0.04369066655635834
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,1,float16,float16,1,0.35891199111938477
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,1,float16,fp8,1,0.2810879945755005
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,1,float16,float16,3,0.36881065368652344
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,1,float16,fp8,3,0.29098665714263916
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,1,float16,float16,7,0.3729066848754883
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,1,float16,fp8,7,0.30190932750701904
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,1,float16,float16,15,0.4466346502304077
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,1,float16,float16,31,0.4490240017573039
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,1,float16,fp8,15,0.38758401075998944
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,1,float16,fp8,31,0.38546133041381836
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,1,float16,float16,63,0.4510720173517863
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,1,float16,fp8,63,0.38894931475321454
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,1,float16,float16,127,0.5382826725641886
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,1,float16,fp8,127,0.4602880080540975
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,1,float16,float16,255,0.8084479967753092
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,1,float16,float16,1,0.35891199111938477
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,1,float16,fp8,255,0.7140639623006185
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,1,float16,fp8,1,0.2810879945755005
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,1,float16,float16,3,0.36881065368652344
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,1,float16,fp8,3,0.2913279930750529
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,1,float16,float16,7,0.3742719888687134
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,1,float16,fp8,7,0.3036213318506877
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,1,float16,float16,15,0.44697598616282147
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,1,float16,fp8,15,0.38758401075998944
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,1,float16,float16,31,0.45140798886617023
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,1,float16,fp8,31,0.3872426748275757
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,1,float16,float16,63,0.45311999320983887
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,1,float16,fp8,63,0.38893866539001465
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,1,float16,fp8,127,0.4613120158513387
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,1,float16,float16,127,0.5393066803614298
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,1,float16,float16,255,0.8077653249104818
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,1,float16,float16,1,0.35891199111938477
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,1,float16,fp8,255,0.7150932947794596
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,1,float16,fp8,1,0.28142933050791424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,1,float16,fp8,3,0.2913279930750529
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,1,float16,float16,3,0.3684693177541097
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,1,float16,float16,7,0.3739306529362996
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,1,float16,fp8,7,0.30532266696294147
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,1,float16,float16,15,0.4469706614812215
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,1,float16,fp8,15,0.3872426748275757
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,1,float16,float16,31,0.4524373213450114
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,1,float16,fp8,31,0.3872426748275757
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,1,float16,float16,63,0.45311999320983887
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,1,float16,fp8,63,0.389631986618042
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,1,float16,float16,127,0.540666659673055
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,1,float16,fp8,127,0.460970679918925
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,1,float16,float16,255,0.866650660832723
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,1,float16,fp8,255,0.7147520383199056
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,1,float16,float16,1,0.04744533201058706
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,1,float16,fp8,1,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,1,float16,float16,3,0.04744533201058706
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,1,float16,fp8,3,0.04437333345413208
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,1,float16,float16,7,0.04744533201058706
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,1,float16,fp8,7,0.043338666359583534
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,1,float16,float16,15,0.04744533201058706
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,1,float16,fp8,15,0.04539733131726583
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,1,float16,float16,31,0.04744533201058706
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,1,float16,fp8,31,0.04334400097529093
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,1,float16,float16,63,0.04710400104522705
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,1,float16,fp8,63,0.04506133496761322
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,1,float16,fp8,15,0.014127999544143677
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,1,float16,float16,127,0.04778666794300079
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,1,float16,fp8,127,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,1,float16,float16,255,0.04983466863632202
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,1,float16,fp8,255,0.041637333730856575
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,1,float16,float16,1,0.7114559809366862
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,1,float16,fp8,1,0.5550080140431722
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,1,float16,float16,3,0.7294293244679769
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,1,float16,fp8,3,0.5765120188395182
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,1,float16,float16,7,0.7376266320546468
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,1,float16,fp8,7,0.5949440002441406
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,1,float16,float16,15,0.885754664738973
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,1,float16,fp8,15,0.7679999669392904
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,1,float16,float16,31,0.8932693004608154
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,1,float16,fp8,31,0.7649280230204264
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,1,float16,float16,63,0.8970239957173666
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,1,float16,fp8,63,0.7703893184661865
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,1,float16,float16,127,1.0712800025939941
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,1,float16,fp8,127,0.9137492974599203
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,1,float16,float16,1,0.7106560071309408
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,1,float16,fp8,1,0.5560373465220133
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,1,float16,float16,3,0.7297653357187907
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,1,float16,fp8,3,0.5761706829071045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,1,float16,float16,7,0.7406933307647705
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,1,float16,fp8,7,0.6017760038375854
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,1,float16,float16,15,0.8874666690826416
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,1,float16,fp8,15,0.7683413028717041
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,1,float16,float16,31,0.897706667582194
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,1,float16,fp8,31,0.7693599859873453
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,1,float16,float16,63,0.9014613628387451
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,1,float16,fp8,63,0.7703893184661865
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,1,float16,fp8,127,0.9154559771219889
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,1,float16,float16,127,1.0920960108439128
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,1,float16,float16,1,0.7106560071309408
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,1,float16,fp8,1,0.5550186634063721
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,1,float16,float16,3,0.7338666915893555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,1,float16,fp8,3,0.5741173426310221
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,1,float16,float16,7,0.7437653541564941
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,1,float16,fp8,7,0.6021066506703695
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,1,float16,float16,15,0.8905279636383057
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,1,float16,fp8,15,0.7666347026824951
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,1,float16,float16,31,0.899066686630249
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,1,float16,fp8,31,0.7659520308176676
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,1,float16,float16,63,0.9180213610331217
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,1,float16,float16,127,1.155242681503296
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,1,float16,fp8,63,0.7707306543986002
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,1,float16,fp8,127,0.9268906911214193
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,1,float16,fp8,1,0.07815999786059062
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,1,float16,float16,1,0.08533333738644917
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,1,float16,float16,3,0.08430932958920796
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,1,float16,fp8,3,0.077824001510938
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,1,float16,float16,7,0.08430400490760803
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,1,float16,fp8,7,0.07815999786059062
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,1,float16,float16,15,0.08430932958920796
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,1,float16,fp8,15,0.07816533247629802
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,1,float16,float16,31,0.0846506655216217
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,1,float16,fp8,31,0.07817066709200542
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,1,float16,float16,63,0.08533333738644917
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,1,float16,float16,1,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,1,float16,fp8,63,0.07816533247629802
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,1,float16,float16,127,0.08635733524958293
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,1,float16,fp8,127,0.07816533247629802
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,1,float16,float16,3,0.01893866683046023
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,1,float16,fp8,1,0.014671999961137772
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,1,float16,fp8,3,0.015589332828919092
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,1,float16,float16,7,0.020725333442290623
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,1,float16,fp8,7,0.014959999670584997
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,1,float16,fp8,15,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,1,float16,float16,15,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,1,float16,float16,31,0.0210506667693456
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,1,float16,float16,63,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,1,float16,fp8,31,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,1,float16,fp8,63,0.01870399961868922
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,1,float16,float16,127,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,1,float16,fp8,127,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,1,float16,float16,255,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,1,float16,fp8,255,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,1,float16,float16,511,0.05359466870625814
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,1,float16,fp8,511,0.047450666626294456
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,1,float16,float16,1023,0.09250666697820027
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,1,float16,fp8,1023,0.08226133386294048
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,1,float16,float16,2047,0.17015467087427774
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,1,float16,fp8,2047,0.1520639955997467
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,1,float16,float16,4095,0.3237546682357788
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,1,float16,fp8,4095,0.2913279930750529
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,1,float16,float16,8191,0.6330026785532633
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,1,float16,fp8,8191,0.5717386802037557
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,1,float16,float16,16383,1.4508320490519206
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,1,float16,fp8,16383,1.2414240042368572
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,1,float16,float16,1,0.018672000616788864
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,1,float16,fp8,1,0.015589332828919092
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,1,float16,float16,3,0.018719999740521114
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,1,float16,fp8,3,0.014949332922697067
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,1,float16,float16,7,0.018842666099468868
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,1,float16,fp8,7,0.016517333686351776
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,1,float16,float16,15,0.02075733368595441
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,1,float16,fp8,15,0.01869333287080129
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,1,float16,float16,31,0.0210506667693456
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,1,float16,fp8,31,0.018661333868900936
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,1,float16,float16,63,0.02075733368595441
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,1,float16,fp8,63,0.01871466636657715
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,1,float16,float16,127,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,1,float16,fp8,127,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,1,float16,float16,255,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,1,float16,fp8,255,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,1,float16,float16,511,0.05358933409055074
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,1,float16,fp8,511,0.04744533201058706
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,1,float16,float16,1023,0.09250133236249287
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,1,float16,fp8,1023,0.08226133386294048
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,1,float16,fp8,2047,0.15189866224924722
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,1,float16,float16,2047,0.17031999429066977
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,1,float16,float16,4095,0.3258026639620463
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,1,float16,fp8,4095,0.2913279930750529
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,1,float16,float16,8191,0.6418879826863607
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,1,float16,fp8,8191,0.5748053391774496
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,1,float16,float16,1,0.019007999449968338
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,1,float16,float16,16383,1.7286826769510906
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,1,float16,fp8,1,0.016613333175579708
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,1,float16,fp8,16383,1.2417759895324707
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,1,float16,float16,3,0.019343999524911244
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,1,float16,fp8,3,0.016554666062196095
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,1,float16,float16,7,0.01966399947802226
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,1,float16,fp8,7,0.01598400001724561
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,1,float16,float16,15,0.02070933332045873
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,1,float16,fp8,15,0.018719999740521114
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,1,float16,float16,31,0.021055998901526134
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,1,float16,fp8,31,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,1,float16,float16,63,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,1,float16,fp8,63,0.018709332992633183
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,1,float16,float16,127,0.02492266645034154
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,1,float16,fp8,127,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,1,float16,float16,255,0.03346133232116699
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,1,float16,fp8,255,0.029343999922275543
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,1,float16,float16,511,0.05358933409055074
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,1,float16,fp8,511,0.04744533201058706
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,1,float16,float16,1023,0.09250133236249287
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,1,float16,fp8,1023,0.08226133386294048
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,1,float16,float16,2047,0.17032533884048462
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,1,float16,fp8,2047,0.1518933375676473
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,1,float16,float16,4095,0.32409600416819256
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,1,float16,fp8,4095,0.2926933368047078
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,1,float16,float16,8191,0.6731093724568685
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,1,float16,fp8,8191,0.5720746517181396
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,1,float16,float16,16383,1.9372320175170898
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,1,float16,float16,1,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,1,float16,fp8,16383,1.32369065284729
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,1,float16,fp8,1,0.012357333054145178
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,1,float16,float16,3,0.012357333054145178
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,1,float16,fp8,3,0.012698666503032049
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,1,float16,fp8,7,0.011194666226704916
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,1,float16,float16,15,0.012357333054145178
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,1,float16,fp8,15,0.01379199946920077
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,1,float16,float16,31,0.01051733394463857
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,1,float16,fp8,31,0.012533333152532578
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,1,float16,float16,63,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,1,float16,fp8,63,0.012362666428089142
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,1,float16,float16,127,0.012533333152532578
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,1,float16,fp8,127,0.013616000612576803
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,1,float16,float16,255,0.012458667159080505
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,1,float16,fp8,255,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,1,float16,float16,511,0.013167999684810638
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,1,float16,fp8,511,0.012800000607967377
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,1,float16,float16,1023,0.014560000350077948
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,1,float16,fp8,1023,0.014576000471909841
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,1,float16,float16,2047,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,1,float16,fp8,2047,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,1,float16,float16,4095,0.022757334013779957
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,1,float16,float16,8191,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,1,float16,fp8,4095,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,1,float16,fp8,8191,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,1,float16,float16,16383,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,1,float16,fp8,16383,0.04266666869322459
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,1,float16,fp8,1,1.1047253608703613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,1,float16,float16,1,1.4230186144510906
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,1,float16,float16,3,1.4603947003682454
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,1,float16,fp8,3,1.1467040379842122
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,1,float16,float16,7,1.4871892929077148
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,1,float16,fp8,7,1.189893325169881
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,1,float16,float16,15,1.765541394551595
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,1,float16,fp8,15,1.5305387179056804
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,1,float16,float16,31,1.787391980489095
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,1,float16,fp8,31,1.5226880709330242
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,1,float16,fp8,63,1.5359999338785808
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,1,float16,float16,63,1.8034292856852214
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,1,float16,float16,1,1.4274667104085286
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,1,float16,fp8,1,1.1152959664662678
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,1,float16,float16,3,1.470469315846761
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,1,float16,fp8,3,1.1511413256327312
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,1,float16,float16,7,1.4916267395019531
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,1,float16,fp8,7,1.2052480379740398
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,1,float16,float16,15,1.7747680346171062
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,1,float16,fp8,15,1.527466615041097
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,1,float16,float16,31,1.8112853368123372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,1,float16,fp8,31,1.5305387179056804
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,1,float16,fp8,63,1.5469226837158203
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,1,float16,float16,63,1.8479785919189453
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,1,float16,float16,1,1.4165387153625488
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,1,float16,fp8,1,1.1112159887949626
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,1,float16,float16,3,1.4776320457458496
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,1,float16,fp8,3,1.1439786752065022
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,1,float16,float16,7,1.5155199368794758
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,1,float16,fp8,7,1.2161706288655598
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,1,float16,float16,15,1.7836373647054036
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,1,float16,fp8,15,1.5339573224385579
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,1,float16,float16,31,1.8496853510538738
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,1,float16,fp8,31,1.5387306213378906
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,1,float16,float16,63,1.885525385538737
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,1,float16,fp8,63,1.5585227012634277
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,1,float16,float16,1,0.15598932902018228
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,1,float16,fp8,1,0.14779733618100485
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,1,float16,float16,3,0.15633066495259604
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,1,float16,fp8,3,0.14727999766667685
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,1,float16,float16,7,0.15769066413243613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,1,float16,fp8,7,0.1474453310171763
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,1,float16,float16,15,0.15684266885121664
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,1,float16,fp8,15,0.1479626695315043
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,1,float16,float16,31,0.15717867016792297
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,1,float16,fp8,31,0.14574399590492249
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,1,float16,float16,63,0.15684800346692404
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,1,float16,fp8,63,0.14643200238545737
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,128,1,float16,float16,1,2.9832480748494468
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,128,1,float16,fp8,1,2.340186595916748
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,128,1,float16,float16,3,3.062272071838379
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,128,1,float16,fp8,3,2.396671930948893
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,128,1,float16,float16,7,3.0950400034586587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,128,1,float16,fp8,7,2.492762724558512
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,128,1,float16,float16,15,3.5703465143839517
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,128,1,float16,fp8,15,3.0824108123779297
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,128,1,float16,float16,31,3.630762736002604
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,128,1,float16,fp8,31,3.07148806254069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,128,1,float16,float16,1,3.026773452758789
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,128,1,float16,fp8,1,2.3625386555989585
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,128,1,float16,float16,3,3.0906025568644204
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,128,1,float16,fp8,3,2.4116907119750977
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,128,1,float16,float16,7,3.128490765889486
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,128,1,float16,fp8,7,2.5176746050516763
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,128,1,float16,float16,15,3.598330815633138
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,128,1,float16,fp8,15,3.094341278076172
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,128,1,float16,float16,31,3.6766719818115234
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,128,1,float16,fp8,31,3.1008478800455728
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,128,1,float16,float16,1,3.0830933252970376
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,128,1,float16,fp8,1,2.400773366292318
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,128,1,float16,float16,3,3.1394131978352866
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,128,1,float16,fp8,3,2.44923734664917
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,128,1,float16,float16,7,3.1667200724283853
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,128,1,float16,fp8,7,2.551637331644694
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,128,1,float16,float16,15,3.6160853703816733
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,128,1,float16,fp8,15,3.105274518330892
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,128,1,float16,fp8,31,3.108698527018229
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,128,1,float16,float16,31,3.727871894836426
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,128,1,float16,float16,1,0.29542932907740277
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,128,1,float16,float16,3,0.29713066418965656
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,128,1,float16,fp8,1,0.2783573269844055
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,128,1,float16,fp8,3,0.27767467498779297
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,128,1,float16,fp8,7,0.2786986629168193
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,128,1,float16,float16,7,0.29712533950805664
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,128,1,float16,float16,15,0.2957599957784017
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,128,1,float16,fp8,15,0.2769920031229655
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,128,1,float16,float16,31,0.29781333605448407
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,1,float16,fp8,1,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,1,float16,float16,1,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,1,float16,float16,3,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,1,float16,fp8,3,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,128,1,float16,fp8,31,0.27665066719055176
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,1,float16,float16,7,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,1,float16,fp8,7,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,1,float16,float16,15,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,1,float16,fp8,15,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,1,float16,float16,31,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,1,float16,fp8,31,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,1,float16,float16,63,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,1,float16,fp8,63,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,1,float16,float16,127,0.04027733455101649
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,1,float16,fp8,127,0.03721066564321518
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,1,float16,float16,255,0.05871466795603434
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,1,float16,fp8,255,0.05153066913286845
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,1,float16,float16,511,0.09659733374913533
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,1,float16,fp8,511,0.08635733524958293
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,1,float16,float16,1023,0.16810667514801025
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,1,float16,fp8,1023,0.15410666664441428
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,1,float16,float16,2047,0.3118079900741577
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,1,float16,fp8,2047,0.29098133246103924
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,1,float16,float16,4095,0.6034773190816244
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,1,float16,fp8,4095,0.5614933172861735
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,1,float16,float16,8191,1.1813653310139973
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,1,float16,fp8,8191,1.1091626485188801
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,1,float16,fp8,1,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,1,float16,float16,1,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,1,float16,float16,3,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,1,float16,float16,7,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,1,float16,fp8,7,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,1,float16,float16,15,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,1,float16,fp8,15,0.031066666046778362
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,1,float16,float16,31,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,1,float16,fp8,31,0.031066666046778362
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,1,float16,float16,63,0.03345066557327906
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,1,float16,fp8,63,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,1,float16,float16,127,0.03925866633653641
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,1,float16,fp8,127,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,1,float16,float16,255,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,1,float16,fp8,255,0.05358933409055074
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,1,float16,float16,511,0.09454933802286784
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,1,float16,fp8,511,0.08635200063387553
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,1,float16,float16,1023,0.16844799121220908
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,1,float16,fp8,1023,0.15582399566968283
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,1,float16,float16,2047,0.3128373424212138
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,1,float16,fp8,2047,0.29098665714263916
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,1,float16,float16,4095,0.6096320152282715
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,1,float16,fp8,4095,0.5625173250834147
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,1,float16,float16,8191,1.2325546741485596
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,1,float16,fp8,8191,1.1088159879048665
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,1,float16,float16,1,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,1,float16,fp8,1,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,1,float16,float16,3,0.029018667836983997
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,1,float16,fp8,3,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,1,float16,float16,7,0.029685333371162415
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,1,float16,fp8,7,0.025600001215934753
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,1,float16,float16,15,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,1,float16,fp8,15,0.03105599929889043
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,1,float16,float16,31,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,1,float16,fp8,31,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,1,float16,float16,63,0.03309866786003113
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,1,float16,fp8,63,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,1,float16,float16,127,0.04027733455101649
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,1,float16,fp8,127,0.03721066564321518
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,1,float16,float16,255,0.0576853354771932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,1,float16,fp8,255,0.05358933409055074
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,1,float16,fp8,511,0.08635733524958293
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,1,float16,float16,511,0.09454933802286784
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,1,float16,float16,1023,0.16844799121220908
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,1,float16,fp8,1023,0.1539413332939148
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,1,float16,float16,2047,0.3155626654624939
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,1,float16,fp8,2047,0.29028799136479694
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,1,float16,float16,4095,0.6306133270263672
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,1,float16,fp8,4095,0.5614933172861735
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,1,float16,float16,8191,1.3701012929280598
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,1,float16,fp8,8191,1.1105173428853352
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,1,float16,float16,1,0.012357333054145178
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,1,float16,fp8,1,0.012560000022252401
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,1,float16,float16,3,0.012869333227475485
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,1,float16,fp8,3,0.012719999998807907
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,1,float16,float16,7,0.012703999876976013
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,1,float16,fp8,7,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,1,float16,float16,15,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,1,float16,fp8,15,0.01268799975514412
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,1,float16,float16,31,0.012351999680201212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,1,float16,fp8,31,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,1,float16,float16,63,0.012869333227475485
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,1,float16,fp8,63,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,1,float16,float16,127,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,1,float16,fp8,127,0.012709333250919977
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,1,float16,float16,255,0.012346666306257248
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,1,float16,fp8,255,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,1,float16,float16,511,0.014629332969586054
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,1,float16,fp8,511,0.012703999876976013
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,1,float16,float16,1023,0.016623999923467636
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,1,float16,fp8,1023,0.01461333284775416
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,1,float16,float16,2047,0.02275199939807256
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,1,float16,fp8,2047,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,1,float16,float16,4095,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,1,float16,fp8,4095,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,1,float16,float16,8191,0.06007466713587443
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,1,float16,fp8,8191,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,1,float16,float16,1,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,1,float16,fp8,1,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,1,float16,float16,3,0.03925866633653641
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,1,float16,fp8,3,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,1,float16,float16,7,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,1,float16,fp8,7,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,1,float16,float16,15,0.04744533201058706
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,1,float16,fp8,15,0.04265599946180979
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,1,float16,float16,31,0.049498667319615684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,1,float16,fp8,31,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,1,float16,float16,63,0.04915733138720194
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,1,float16,fp8,63,0.043354665239652
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,1,float16,float16,127,0.0576800008614858
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,1,float16,fp8,127,0.050186668833096824
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,1,float16,float16,255,0.0846560001373291
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,1,float16,fp8,255,0.07577600081761678
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,1,float16,float16,511,0.13944000005722046
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,1,float16,float16,1023,0.24815466006596884
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,1,float16,fp8,511,0.12526933352152506
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,1,float16,fp8,1023,0.22766933838526407
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,1,float16,float16,2047,0.4613120158513387
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,1,float16,fp8,2047,0.42820799350738525
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,1,float16,float16,4095,0.8932693004608154
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,1,float16,fp8,4095,0.8320000171661377
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,1,float16,float16,7,0.01119999960064888
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,1,float16,fp8,3,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,1,float16,float16,1,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,1,float16,fp8,1,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,1,float16,float16,3,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,1,float16,fp8,3,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,1,float16,float16,7,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,1,float16,fp8,7,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,1,float16,fp8,15,0.043007999658584595
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,1,float16,float16,15,0.04744533201058706
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,1,float16,float16,31,0.04744533201058706
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,1,float16,fp8,31,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,1,float16,float16,63,0.04744533201058706
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,1,float16,fp8,63,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,1,float16,float16,127,0.0576853354771932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,1,float16,fp8,127,0.051541333397229515
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,1,float16,fp8,255,0.07611733178297679
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,1,float16,float16,255,0.08499733606974284
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,1,float16,float16,511,0.13944000005722046
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,1,float16,fp8,511,0.12526933352152506
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,1,float16,float16,1023,0.2481493353843689
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,1,float16,fp8,1023,0.22561599810918173
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,1,float16,float16,2047,0.48110934098561603
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,1,float16,fp8,2047,0.42820799350738525
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,1,float16,float16,4095,1.0156373182932537
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,1,float16,fp8,4095,0.8347307046254476
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,1,float16,float16,1,0.01524266724785169
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,1,float16,fp8,1,0.01451733335852623
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,1,float16,float16,3,0.014757333944241205
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,1,float16,fp8,3,0.014831999937693277
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,1,float16,float16,7,0.01661866654952367
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,1,float16,fp8,7,0.014607999473810196
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,1,float16,float16,15,0.014767999450365702
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,1,float16,fp8,15,0.01461333284775416
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,1,float16,float16,31,0.016607999801635742
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,1,float16,fp8,31,0.014602666099866232
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,1,float16,fp8,63,0.014736000448465347
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,1,float16,float16,127,0.014848000059525171
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,1,float16,fp8,127,0.014853333433469137
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,1,float16,float16,255,0.014858666807413101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,1,float16,fp8,255,0.014511999984582266
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,1,float16,float16,511,0.01700266698996226
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,1,float16,fp8,511,0.01672533278663953
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,1,float16,float16,1023,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,1,float16,fp8,1023,0.018768000106016796
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,1,float16,float16,2047,0.04061333338419596
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,1,float16,fp8,2047,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,1,float16,float16,4095,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,1,float16,fp8,4095,0.04539733131726583
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,1,float16,float16,1,0.010837333897749582
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,1,float16,fp8,1,0.00983466642598311
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,1,float16,float16,3,0.009786666681369146
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,1,float16,fp8,3,0.00972800018886725
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,1,float16,float16,7,0.010069333637754122
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,1,float16,fp8,7,0.009999999776482582
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,1,float16,float16,15,0.010154666379094124
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,1,float16,fp8,15,0.010698666175206503
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,1,float16,float16,31,0.01198400060335795
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,1,float16,fp8,31,0.010341333225369453
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,1,float16,float16,63,0.010501333822806677
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,1,float16,float16,127,0.010928000013033548
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,1,float16,fp8,127,0.010581333190202713
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,1,float16,fp8,63,0.011440000186363855
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,1,float16,float16,255,0.022197333474953968
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,1,float16,fp8,255,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,1,float16,float16,511,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,1,float16,fp8,511,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,1,float16,float16,1023,0.031744000812371574
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,1,float16,fp8,1023,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,1,float16,fp8,2047,0.041306667029857635
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,1,float16,float16,2047,0.04540266593297323
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,1,float16,float16,4095,0.07064533233642578
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,1,float16,fp8,4095,0.06587733328342438
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,1,float16,float16,8191,0.12117333213488261
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,1,float16,fp8,8191,0.1109333336353302
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,1,float16,float16,16383,0.22357332706451416
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,1,float16,fp8,16383,0.20121600230534872
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,1,float16,float16,63,0.016634666671355564
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,1,float16,float16,1,0.010458666831254959
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,1,float16,fp8,1,0.010431999961535135
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,1,float16,float16,3,0.010426666587591171
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,1,float16,fp8,3,0.01032533310353756
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,1,float16,float16,7,0.010581333190202713
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,1,float16,fp8,7,0.010319999729593595
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,1,float16,float16,15,0.011610666910807291
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,1,float16,fp8,15,0.010010666524370512
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,1,float16,float16,31,0.010405333091815313
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,1,float16,fp8,31,0.010575999816258749
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,1,float16,float16,63,0.011776000261306763
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,1,float16,fp8,63,0.011776000261306763
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,1,float16,float16,127,0.011946666985750198
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,1,float16,fp8,127,0.011663999408483505
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,1,float16,float16,255,0.022863999009132385
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,1,float16,fp8,255,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,1,float16,float16,511,0.026288000245889027
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,1,float16,fp8,511,0.024853333830833435
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,1,float16,float16,2047,0.04539733131726583
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,1,float16,fp8,1023,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,1,float16,fp8,2047,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,1,float16,float16,4095,0.07202133536338806
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,1,float16,float16,1023,0.03276800115903219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,1,float16,fp8,4095,0.06587733328342438
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,1,float16,float16,8191,0.1232266624768575
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,1,float16,fp8,8191,0.1109386682510376
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,1,float16,float16,16383,0.22528000672658285
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,1,float16,fp8,16383,0.20309333006540933
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,1,float16,float16,1,0.012469333906968435
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,1,float16,fp8,1,0.012485332787036896
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,1,float16,float16,3,0.01099733387430509
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,1,float16,float16,7,0.012784000486135483
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,1,float16,fp8,7,0.012357333054145178
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,1,float16,fp8,3,0.012458667159080505
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,1,float16,float16,15,0.012357333054145178
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,1,float16,fp8,15,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,1,float16,float16,31,0.012272000312805176
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,1,float16,fp8,31,0.01228800043463707
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,1,float16,float16,63,0.012351999680201212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,1,float16,fp8,63,0.012975999464591345
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,1,float16,float16,127,0.010672000547250112
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,1,float16,fp8,127,0.010666667173306147
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,1,float16,float16,511,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,1,float16,fp8,255,0.010687999427318573
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,1,float16,float16,255,0.013194666554530462
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,1,float16,fp8,511,0.013056000073750814
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,1,float16,float16,1023,0.012400000045696894
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,1,float16,fp8,1023,0.01302933320403099
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,1,float16,float16,2047,0.012869333227475485
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,1,float16,fp8,2047,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,1,float16,float16,4095,0.016613333175579708
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,1,float16,fp8,4095,0.01695999999841054
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,1,float16,float16,8191,0.02041600023706754
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,1,float16,float16,16383,0.022757334013779957
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,1,float16,fp8,16383,0.02075733368595441
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,1,float16,fp8,8191,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,1,float16,float16,1,0.014677333335081736
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,1,float16,fp8,1,0.016656000167131424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,1,float16,float16,3,0.014416000495354334
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,1,float16,fp8,3,0.016602666427691776
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,1,float16,float16,7,0.014607999473810196
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,1,float16,fp8,7,0.01681600014368693
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,1,float16,float16,15,0.01479999969402949
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,1,float16,fp8,15,0.016805333395799
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,1,float16,fp8,31,0.0186666672428449
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,1,float16,float16,63,0.01666133354107539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,1,float16,float16,31,0.016805333395799
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,1,float16,fp8,63,0.02276266614596049
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,1,float16,float16,127,0.01672533278663953
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,1,float16,fp8,127,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,1,float16,float16,255,0.018661333868900936
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,1,float16,fp8,255,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,1,float16,float16,511,0.02480533222357432
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,1,float16,fp8,511,0.024911999702453613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,1,float16,float16,1023,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,1,float16,fp8,1023,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,1,float16,float16,2047,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,1,float16,fp8,2047,0.05358933409055074
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,1,float16,fp8,4095,0.09045333663622539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,1,float16,float16,8191,0.20309333006540933
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,1,float16,float16,4095,0.1088853379090627
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,1,float16,fp8,8191,0.16435733437538147
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,1,float16,float16,16383,0.3933866818745931
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,1,float16,fp8,16383,0.31249066193898517
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,1,float16,float16,1,0.014576000471909841
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,1,float16,fp8,1,0.01661866654952367
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,1,float16,float16,3,0.014570667097965876
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,1,float16,fp8,3,0.016613333175579708
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,1,float16,float16,7,0.014570667097965876
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,1,float16,fp8,7,0.016666666915019352
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,1,float16,float16,15,0.014762666076421738
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,1,float16,fp8,15,0.01695466662446658
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,1,float16,float16,31,0.01661866654952367
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,1,float16,fp8,31,0.018768000106016796
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,1,float16,float16,63,0.016602666427691776
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,1,float16,fp8,63,0.02184533327817917
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,1,float16,float16,127,0.016613333175579708
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,1,float16,fp8,127,0.02218666672706604
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,1,float16,float16,255,0.018778666853904724
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,1,float16,fp8,255,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,1,float16,float16,511,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,1,float16,fp8,511,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,1,float16,fp8,1023,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,1,float16,float16,2047,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,1,float16,float16,1023,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,1,float16,float16,4095,0.1088853379090627
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,1,float16,fp8,4095,0.09045333663622539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,1,float16,float16,8191,0.2053119937578837
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,1,float16,fp8,2047,0.05358933409055074
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,1,float16,fp8,8191,0.1641866664091746
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,1,float16,float16,16383,0.3959466616312663
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,1,float16,fp8,16383,0.3135146697362264
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,1,float16,float16,1,0.010687999427318573
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,1,float16,fp8,1,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,1,float16,float16,3,0.01081066702802976
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,1,float16,fp8,3,0.010863999525705973
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,1,float16,float16,15,0.010874666273593903
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,1,float16,float16,7,0.01080000028014183
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,1,float16,fp8,7,0.011178666104873022
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,1,float16,fp8,15,0.010863999525705973
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,1,float16,float16,31,0.012367999802033106
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,1,float16,fp8,31,0.011157333850860596
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,1,float16,float16,63,0.01102399950226148
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,1,float16,fp8,63,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,1,float16,float16,127,0.012357333054145178
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,1,float16,fp8,127,0.010693332801262537
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,1,float16,float16,255,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,1,float16,fp8,255,0.011274666835864386
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,1,float16,float16,511,0.012479999413092932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,1,float16,fp8,511,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,1,float16,float16,1023,0.01251199965675672
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,1,float16,fp8,1023,0.012703999876976013
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,1,float16,float16,2047,0.016949333250522614
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,1,float16,fp8,2047,0.01681600014368693
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,1,float16,float16,4095,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,1,float16,fp8,4095,0.0186666672428449
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,1,float16,float16,8191,0.02072000006834666
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,1,float16,fp8,8191,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,1,float16,float16,16383,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,1,float16,fp8,16383,0.02388266722361247
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,1,float16,float16,1,0.07202666501204173
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,1,float16,fp8,1,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,1,float16,float16,3,0.07406933108965556
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,1,float16,fp8,3,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,1,float16,float16,7,0.07611733178297679
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,1,float16,fp8,7,0.06382933259010315
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,1,float16,fp8,15,0.07816533247629802
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,1,float16,float16,15,0.08840533097585042
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,1,float16,float16,31,0.09045333663622539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,1,float16,fp8,31,0.07816533247629802
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,1,float16,float16,63,0.09045333663622539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,1,float16,fp8,63,0.07816533247629802
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,1,float16,fp8,127,0.09250133236249287
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,1,float16,float16,127,0.10683199763298035
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,1,float16,float16,255,0.15821333726247153
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,1,float16,fp8,511,0.2379093368848165
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,1,float16,float16,511,0.2643679976463318
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,1,float16,float16,1023,0.4715520143508911
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,1,float16,fp8,1023,0.43298133214314777
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,1,float16,float16,2047,0.8878080050150553
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,1,float16,fp8,255,0.14165332913398743
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,1,float16,fp8,2047,0.8231306870778402
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,1,float16,float16,1,0.07406933108965556
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,1,float16,fp8,1,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,1,float16,float16,3,0.07406933108965556
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,1,float16,fp8,3,0.06178666651248932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,1,float16,float16,7,0.07611733178297679
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,1,float16,float16,15,0.08841066559155782
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,1,float16,fp8,15,0.07816533247629802
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,1,float16,fp8,7,0.06382933259010315
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,1,float16,float16,31,0.09045333663622539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,1,float16,fp8,31,0.07816533247629802
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,1,float16,float16,63,0.09045333663622539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,1,float16,fp8,63,0.07816533247629802
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,1,float16,fp8,127,0.09250133236249287
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,1,float16,float16,127,0.1088853379090627
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,1,float16,float16,255,0.1599146624406179
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,1,float16,fp8,255,0.14148267110188803
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,1,float16,float16,511,0.264357328414917
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,1,float16,fp8,511,0.2379093368848165
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,1,float16,float16,1023,0.5009066661198934
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,1,float16,fp8,1023,0.43263999621073407
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,1,float16,float16,2047,1.0036959648132324
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,1,float16,fp8,2047,0.8268640041351318
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,1,float16,float16,1,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,1,float16,fp8,1,0.018661333868900936
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,1,float16,float16,3,0.020143999407688778
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,1,float16,fp8,3,0.01865600049495697
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,1,float16,float16,7,0.02075733368595441
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,1,float16,fp8,7,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,1,float16,float16,15,0.01871466636657715
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,1,float16,fp8,15,0.0186666672428449
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,1,float16,float16,31,0.02075733368595441
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,1,float16,fp8,31,0.018677332748969395
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,1,float16,float16,63,0.01871466636657715
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,1,float16,fp8,63,0.0186666672428449
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,1,float16,float16,127,0.018709332992633183
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,1,float16,fp8,127,0.018677332748969395
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,1,float16,float16,255,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,1,float16,fp8,255,0.01782400036851565
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,1,float16,float16,511,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,1,float16,fp8,511,0.021157334248224895
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,1,float16,float16,1023,0.03719999889532725
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,1,float16,fp8,1023,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,1,float16,float16,2047,0.062447999914487205
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,1,float16,fp8,2047,0.045738667249679565
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,1,float16,float16,1,0.018944000204404194
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,1,float16,fp8,1,0.014885333677132925
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,1,float16,float16,3,0.018709332992633183
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,1,float16,fp8,3,0.014912000546852747
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,1,float16,float16,7,0.018698666244745255
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,1,float16,fp8,7,0.016469333320856094
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,1,float16,float16,15,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,1,float16,fp8,15,0.016469333320856094
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,1,float16,float16,31,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,1,float16,fp8,31,0.016821333517630894
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,1,float16,float16,63,0.022800001005331676
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,1,float16,fp8,63,0.016949333250522614
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,1,float16,float16,127,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,1,float16,fp8,127,0.018661333868900936
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,1,float16,float16,255,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,1,float16,fp8,255,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,1,float16,float16,511,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,1,float16,fp8,511,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,1,float16,float16,1023,0.05358933409055074
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,1,float16,fp8,1023,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,1,float16,float16,2047,0.08841066559155782
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,1,float16,fp8,2047,0.07065600156784058
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,1,float16,float16,4095,0.16025599837303162
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,1,float16,fp8,4095,0.12936000029246011
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,1,float16,float16,8191,0.3036160071690877
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,1,float16,fp8,8191,0.24474134047826132
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,1,float16,float16,16383,0.5911839803059896
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,1,float16,fp8,16383,0.4777013460795085
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,1,float16,float16,1,0.019002666076024372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,1,float16,fp8,1,0.014762666076421738
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,1,float16,float16,3,0.018768000106016796
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,1,float16,fp8,3,0.01661866654952367
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,1,float16,float16,7,0.01899733394384384
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,1,float16,fp8,7,0.016623999923467636
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,1,float16,float16,15,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,1,float16,fp8,15,0.016613333175579708
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,1,float16,float16,31,0.022874665757020313
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,1,float16,fp8,31,0.018432000031073887
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,1,float16,float16,63,0.022863999009132385
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,1,float16,fp8,63,0.016943999876578648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,1,float16,float16,127,0.022858666876951855
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,1,float16,fp8,127,0.01729600007335345
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,1,float16,float16,255,0.02697066714366277
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,1,float16,fp8,255,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,1,float16,float16,511,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,1,float16,fp8,511,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,1,float16,float16,1023,0.05358933409055074
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,1,float16,fp8,1023,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,1,float16,float16,2047,0.09045333663622539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,1,float16,fp8,2047,0.07202133536338806
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,1,float16,float16,4095,0.16196266810099283
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,1,float16,fp8,4095,0.1293653349081675
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,1,float16,float16,8191,0.3063466747601827
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,1,float16,fp8,8191,0.24576000372568765
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,1,float16,float16,16383,0.6089333295822144
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,1,float16,fp8,16383,0.4773546854654948
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,1,float16,fp8,1,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,1,float16,float16,1,0.01119999960064888
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,1,float16,float16,3,0.012506666282812754
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,1,float16,float16,7,0.012709333250919977
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,1,float16,fp8,3,0.01051733394463857
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,1,float16,fp8,7,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,1,float16,float16,15,0.010522666076819101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,1,float16,fp8,15,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,1,float16,float16,31,0.011792000383138657
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,1,float16,float16,63,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,1,float16,fp8,63,0.01051733394463857
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,1,float16,float16,127,0.010656000425418219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,1,float16,fp8,127,0.010522666076819101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,1,float16,float16,255,0.010522666076819101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,1,float16,fp8,255,0.010687999427318573
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,1,float16,float16,511,0.012554666648308435
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,1,float16,fp8,511,0.012560000022252401
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,1,float16,float16,1023,0.013125333935022354
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,1,float16,fp8,1023,0.012800000607967377
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,1,float16,float16,2047,0.016656000167131424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,1,float16,fp8,2047,0.016656000167131424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,1,float16,float16,4095,0.018709332992633183
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,1,float16,fp8,4095,0.01672533278663953
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,1,float16,float16,8191,0.02218666672706604
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,1,float16,fp8,8191,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,1,float16,fp8,16383,0.02696000039577484
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,1,float16,float16,16383,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,1,float16,float16,1,0.13960533340771994
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,1,float16,float16,3,0.1418239971001943
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,1,float16,fp8,1,0.1109333336353302
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,1,float16,fp8,3,0.1129813293615977
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,1,float16,float16,7,0.14574933052062988
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,1,float16,fp8,7,0.11912533640861511
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,1,float16,float16,15,0.1723733345667521
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,1,float16,fp8,15,0.14933866262435913
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,1,float16,float16,31,0.17254932721455893
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,1,float16,fp8,31,0.14967466394106546
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,1,float16,float16,63,0.17442133029301962
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,1,float16,fp8,63,0.14967466394106546
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,1,float16,float16,127,0.20718934138615927
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,1,float16,fp8,127,0.17665066321690878
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,1,float16,float16,255,0.3094186584154765
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,1,float16,fp8,255,0.2735840082168579
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,1,float16,float16,511,0.5159253279368082
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,1,float16,fp8,511,0.4633493423461914
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,1,float16,float16,1023,0.9251840114593506
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,1,float16,fp8,1023,0.846506675084432
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,1,float16,fp8,31,0.012186666329701742
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,1,float16,float16,1,0.13960533340771994
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,1,float16,fp8,1,0.1109333336353302
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,1,float16,float16,3,0.1418239971001943
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,1,float16,fp8,3,0.11366400122642517
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,1,float16,float16,7,0.1437066694100698
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,1,float16,fp8,7,0.11912000179290771
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,1,float16,float16,15,0.1723733345667521
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,1,float16,fp8,15,0.14984533190727234
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,1,float16,float16,31,0.17425066232681274
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,1,float16,fp8,31,0.14967466394106546
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,1,float16,float16,63,0.1742453376452128
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,1,float16,fp8,63,0.15001599987347922
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,1,float16,float16,127,0.20736000935236612
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,1,float16,fp8,127,0.1771519978841146
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,1,float16,float16,255,0.3094240029652913
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,1,float16,fp8,255,0.27290133635203045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,1,float16,fp8,511,0.4613066514333089
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,1,float16,float16,1023,1.0350933074951172
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,1,float16,float16,511,0.5447680155436198
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,1,float16,fp8,1023,0.8519786993662516
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,1,float16,float16,1,0.027295999228954315
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,1,float16,fp8,1,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,1,float16,fp8,3,0.02697066714366277
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,1,float16,float16,7,0.02867199977238973
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,1,float16,fp8,7,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,1,float16,float16,15,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,1,float16,fp8,15,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,1,float16,fp8,31,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,1,float16,float16,63,0.027994667490323383
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,1,float16,float16,31,0.027306665976842243
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,1,float16,fp8,63,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,1,float16,float16,127,0.026943999032179516
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,1,float16,fp8,127,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,1,float16,float16,255,0.027306665976842243
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,1,float16,fp8,255,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,1,float16,float16,511,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,1,float16,fp8,511,0.0321066677570343
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,1,float16,float16,1023,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,1,float16,fp8,1023,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,1,float16,float16,1,0.27084799607594806
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,1,float16,fp8,1,0.21333332856496176
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,1,float16,float16,3,0.27801599105199176
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,1,float16,fp8,3,0.2208319902420044
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,1,float16,float16,7,0.28245333830515545
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,1,float16,fp8,7,0.22971733411153158
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,1,float16,float16,15,0.33639466762542725
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,1,float16,fp8,15,0.2913279930750529
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,1,float16,float16,31,0.34013867378234863
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,1,float16,fp8,31,0.29132266839345294
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,1,float16,float16,63,0.3421866496404012
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,1,float16,fp8,63,0.29337600866953534
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,1,float16,float16,127,0.40755200386047363
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,1,float16,fp8,127,0.3466240167617798
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,1,float16,float16,255,0.6085973183314005
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,1,float16,fp8,255,0.5372693141301473
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,1,float16,float16,3,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,1,float16,float16,1,0.2725546757380168
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,1,float16,fp8,1,0.21333332856496176
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,1,float16,float16,3,0.2800640066464742
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,1,float16,fp8,3,0.2194826602935791
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,1,float16,float16,7,0.28279467423756915
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,1,float16,fp8,7,0.23176532983779907
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,1,float16,float16,15,0.33774932225545246
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,1,float16,fp8,15,0.2926933368047078
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,1,float16,float16,31,0.3404800097147624
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,1,float16,fp8,31,0.29133333762486774
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,1,float16,float16,63,0.34252798557281494
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,1,float16,fp8,63,0.29338133335113525
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,1,float16,float16,127,0.4082346757253011
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,1,float16,fp8,127,0.34833598136901855
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,1,float16,float16,255,0.6625279982884725
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,1,float16,fp8,255,0.5379413366317749
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,1,float16,float16,1,0.04539733131726583
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,1,float16,fp8,1,0.04164266586303711
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,1,float16,float16,3,0.04539200166861216
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,1,float16,fp8,3,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,1,float16,fp8,7,0.041306667029857635
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,1,float16,float16,7,0.04539733131726583
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,1,float16,float16,15,0.04539733131726583
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,1,float16,fp8,15,0.042319998145103455
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,1,float16,float16,31,0.04369066655635834
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,1,float16,fp8,31,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,1,float16,float16,63,0.04539733131726583
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,1,float16,fp8,63,0.041989331444104515
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,1,float16,fp8,127,0.041296000281969704
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,1,float16,float16,255,0.04710400104522705
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,1,float16,fp8,255,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,1,float16,float16,1,0.5372586647669474
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,1,float16,fp8,1,0.42001068592071533
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,1,float16,float16,3,0.5505760113398234
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,1,float16,fp8,3,0.43537068367004395
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,1,float16,float16,7,0.5580746730168661
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,1,float16,float16,15,0.6696960131327311
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,1,float16,fp8,15,0.5782186587651571
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,1,float16,fp8,7,0.4500480095545451
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,1,float16,float16,31,0.6761813163757324
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,1,float16,fp8,31,0.5762826601664225
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,1,float16,float16,63,0.6782293319702148
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,1,float16,fp8,63,0.5809493462244669
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,1,float16,float16,127,0.8084479967753092
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,1,float16,fp8,127,0.687445322672526
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,1,float16,float16,127,0.04539733131726583
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,1,float16,float16,1,0.5362346569697062
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,1,float16,fp8,1,0.4184746742248535
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,1,float16,float16,3,0.5492053429285685
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,1,float16,fp8,3,0.43263999621073407
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,1,float16,float16,7,0.559445341428121
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,1,float16,fp8,7,0.45311999320983887
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,1,float16,float16,15,0.667306661605835
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,1,float16,fp8,15,0.5775306622187296
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,1,float16,float16,31,0.6751573085784912
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,1,float16,fp8,31,0.5761653184890747
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,1,float16,float16,63,0.6785706679026285
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,1,float16,fp8,63,0.5802666743596395
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,1,float16,float16,127,0.8837119738260905
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,1,float16,fp8,127,0.6905173460642496
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,1,float16,float16,1,0.07884266475836436
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,1,float16,fp8,1,0.07406933108965556
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,1,float16,float16,3,0.07850666840871175
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,1,float16,fp8,3,0.07406933108965556
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,1,float16,float16,7,0.08021333316961925
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,1,float16,fp8,7,0.07406933108965556
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,1,float16,float16,15,0.08020799855391185
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,1,float16,fp8,15,0.07412266731262207
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,1,float16,float16,31,0.08021333316961925
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,1,float16,fp8,31,0.07406400144100189
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,1,float16,float16,63,0.08021333316961925
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,1,float16,fp8,63,0.07406933108965556
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,1,float16,float16,127,0.08260266482830048
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,1,float16,fp8,127,0.07406933108965556
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,1,float16,float16,1,0.014432000617186228
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,1,float16,fp8,1,0.01444799949725469
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,1,float16,float16,3,0.014933332800865173
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,1,float16,fp8,3,0.014954666296641031
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,1,float16,float16,15,0.016832000265518825
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,1,float16,float16,7,0.017765333255132038
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,1,float16,fp8,15,0.016501333564519882
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,1,float16,fp8,7,0.016943999876578648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,1,float16,float16,31,0.018432000031073887
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,1,float16,fp8,31,0.016832000265518825
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,1,float16,float16,63,0.016656000167131424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,1,float16,fp8,63,0.016447999825080235
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,1,float16,float16,127,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,1,float16,fp8,127,0.018837332725524902
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,1,float16,float16,255,0.02890666574239731
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,1,float16,fp8,255,0.025941332181294758
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,1,float16,float16,511,0.043338666359583534
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,1,float16,fp8,511,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,1,float16,float16,1023,0.07611733178297679
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,1,float16,fp8,1023,0.07065066695213318
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,1,float16,float16,2047,0.13960533340771994
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,1,float16,fp8,2047,0.13141333063443503
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,1,float16,float16,4095,0.26470400889714557
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,1,float16,fp8,4095,0.25224532683690387
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,1,float16,fp8,8191,0.49237334728240967
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,1,float16,float16,8191,0.5183200041453043
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,1,float16,float16,16383,1.1624106566111247
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,1,float16,fp8,16383,1.0350933074951172
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,1,float16,float16,1,0.01463466634353002
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,1,float16,fp8,1,0.014671999961137772
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,1,float16,float16,3,0.014896000425020853
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,1,float16,fp8,3,0.01463466634353002
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,1,float16,float16,7,0.014917333920796713
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,1,float16,fp8,7,0.014917333920796713
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,1,float16,float16,15,0.017290666699409485
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,1,float16,fp8,15,0.01666133354107539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,1,float16,fp8,31,0.016672000288963318
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,1,float16,float16,31,0.016997333616018295
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,1,float16,float16,63,0.016943999876578648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,1,float16,float16,127,0.020714666694402695
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,1,float16,fp8,63,0.01672533278663953
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,1,float16,fp8,127,0.018677332748969395
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,1,float16,float16,255,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,1,float16,fp8,255,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,1,float16,float16,511,0.043354665239652
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,1,float16,fp8,511,0.04131199916203817
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,1,float16,float16,1023,0.07611733178297679
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,1,float16,fp8,1023,0.07202133536338806
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,1,float16,float16,2047,0.13943466544151306
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,1,float16,fp8,2047,0.13141333063443503
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,1,float16,float16,4095,0.26470400889714557
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,1,float16,fp8,4095,0.25224532683690387
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,1,float16,float16,8191,0.520362655321757
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,1,float16,fp8,8191,0.4927146832148234
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,1,float16,fp8,16383,1.0426026980082195
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,1,float16,float16,1,0.012351999680201212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,1,float16,fp8,1,0.011546666423479715
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,1,float16,float16,16383,1.510741392771403
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,1,float16,float16,3,0.014389333625634512
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,1,float16,fp8,3,0.01239466667175293
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,1,float16,float16,7,0.012773333738247553
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,1,float16,fp8,7,0.014111999422311783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,1,float16,float16,15,0.011776000261306763
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,1,float16,float16,31,0.01470400020480156
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,1,float16,fp8,15,0.01219733307758967
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,1,float16,fp8,31,0.012714666624863943
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,1,float16,float16,63,0.012800000607967377
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,1,float16,fp8,63,0.011952000359694162
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,1,float16,float16,127,0.0122079998254776
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,1,float16,fp8,127,0.01268799975514412
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,1,float16,float16,255,0.013157332936922709
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,1,float16,fp8,255,0.01250133290886879
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,1,float16,float16,511,0.014570667097965876
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,1,float16,fp8,511,0.013647999614477158
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,1,float16,float16,1023,0.014783999572197596
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,1,float16,fp8,1023,0.014405333747466406
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,1,float16,float16,2047,0.018725333114465077
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,1,float16,fp8,2047,0.018592000007629395
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,1,float16,float16,4095,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,1,float16,fp8,4095,0.02075733368595441
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,1,float16,float16,8191,0.038912000755469
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,1,float16,fp8,8191,0.02696000039577484
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,1,float16,float16,16383,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,1,float16,fp8,16383,0.04199466605981191
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,1,float16,float16,1,1.0682026545206706
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,1,float16,fp8,1,0.831658681233724
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,1,float16,float16,3,1.091754674911499
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,1,float16,fp8,3,0.8645973205566406
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,1,float16,float16,7,1.109503984451294
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,1,float16,fp8,7,0.8919040362040201
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,1,float16,float16,15,1.3335893948872883
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,1,float16,fp8,15,1.1490986347198486
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,1,float16,float16,31,1.3445067405700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,1,float16,fp8,31,1.1480693022410076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,1,float16,float16,63,1.3550933202107747
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,1,float16,fp8,63,1.1531946659088135
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,1,float16,float16,1,1.067530632019043
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,1,float16,fp8,1,0.8299520015716553
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,1,float16,float16,3,1.0948266983032227
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,1,float16,fp8,3,0.8601653575897217
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,1,float16,float16,7,1.1163307030995686
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,1,float16,fp8,7,0.9017919699350992
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,1,float16,float16,15,1.3339306513468425
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,1,float16,fp8,15,1.1477333704630535
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,1,float16,float16,31,1.3875199953715007
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,1,float16,float16,63,1.435306708017985
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,1,float16,fp8,63,1.1743573347727458
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,1,float16,fp8,31,1.1521759827931721
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,1,float16,float16,1,0.1493333379427592
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,1,float16,fp8,1,0.1397813359896342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,1,float16,float16,3,0.14899200201034546
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,1,float16,fp8,3,0.14011200269063315
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,1,float16,float16,7,0.14849066734313965
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,1,float16,float16,15,0.14899200201034546
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,1,float16,fp8,7,0.13959999879201254
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,1,float16,fp8,15,0.1397760013739268
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,1,float16,float16,31,0.14847999811172485
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,1,float16,fp8,31,0.14011733730634054
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,1,float16,float16,63,0.14898133277893066
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,1,float16,fp8,63,0.14147733648618063
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,128,1,float16,float16,1,2.209791978200277
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,128,1,float16,fp8,1,1.7259519894917805
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,128,1,float16,float16,3,2.2836853663126626
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,128,1,float16,fp8,3,1.7877333958943684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,128,1,float16,fp8,7,1.8602666854858398
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,128,1,float16,float16,7,2.3072427113850913
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,128,1,float16,float16,15,2.6808319091796875
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,128,1,float16,fp8,15,2.3145813941955566
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,128,1,float16,float16,31,2.7438027064005532
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,128,1,float16,fp8,31,2.3079253832499185
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,128,1,float16,float16,1,2.3162879943847656
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,128,1,float16,fp8,1,1.7959094047546387
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,128,1,float16,float16,3,2.377216021219889
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,128,1,float16,fp8,3,1.8338027000427246
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,128,1,float16,float16,7,2.408618609110514
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,128,1,float16,fp8,7,1.9130080540974934
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,128,1,float16,float16,15,2.731178601582845
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,128,1,float16,fp8,15,2.3367679913838706
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,128,1,float16,float16,31,2.8276052474975586
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,128,1,float16,fp8,31,2.3391572634379068
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,128,1,float16,float16,1,0.28177066644032794
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,128,1,float16,fp8,1,0.26606933275858563
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,128,1,float16,float16,3,0.28177066644032794
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,128,1,float16,fp8,3,0.2650453249613444
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,128,1,float16,float16,7,0.2821013331413269
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,128,1,float16,fp8,7,0.26641066869099933
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,128,1,float16,float16,15,0.28142933050791424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,128,1,float16,fp8,15,0.2664053241411845
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,128,1,float16,float16,31,0.2821120023727417
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,128,1,float16,fp8,31,0.26573334137598675
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,1,float16,float16,1,0.02492266645034154
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,1,float16,fp8,1,0.019461333751678467
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,1,float16,float16,3,0.02480533222357432
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,1,float16,fp8,3,0.02075200031201045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,1,float16,float16,7,0.02480533222357432
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,1,float16,fp8,7,0.02075200031201045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,1,float16,float16,15,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,1,float16,fp8,15,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,1,float16,float16,31,0.027301333844661713
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,1,float16,fp8,31,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,1,float16,float16,63,0.026975999275843304
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,1,float16,fp8,63,0.024864000578721363
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,1,float16,float16,127,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,1,float16,fp8,127,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,1,float16,float16,255,0.04744533201058706
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,1,float16,fp8,255,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,1,float16,float16,511,0.07611733178297679
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,1,float16,fp8,511,0.06997333467006683
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,1,float16,float16,1023,0.13345600167910257
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,1,float16,fp8,1023,0.12322133779525757
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,1,float16,float16,2047,0.24814399083455405
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,1,float16,fp8,2047,0.22971733411153158
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,1,float16,float16,4095,0.4756480058034261
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,1,float16,fp8,4095,0.44492801030476886
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,1,float16,float16,8191,0.9326933224995931
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,1,float16,fp8,8191,0.8765439987182617
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,1,float16,float16,1,0.024570666253566742
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,1,float16,fp8,1,0.019039999693632126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,1,float16,float16,3,0.02492266645034154
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,1,float16,fp8,3,0.02075733368595441
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,1,float16,float16,7,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,1,float16,fp8,7,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,1,float16,float16,15,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,1,float16,fp8,15,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,1,float16,float16,31,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,1,float16,fp8,31,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,1,float16,float16,63,0.02867199977238973
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,1,float16,fp8,63,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,1,float16,float16,127,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,1,float16,fp8,127,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,1,float16,float16,255,0.04744533201058706
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,1,float16,fp8,255,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,1,float16,float16,511,0.07611733178297679
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,1,float16,fp8,511,0.06997333467006683
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,1,float16,float16,1023,0.13363200426101685
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,1,float16,fp8,1023,0.12322133779525757
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,1,float16,float16,2047,0.2481493353843689
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,1,float16,fp8,2047,0.23142399390538534
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,1,float16,float16,4095,0.47837332884470624
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,1,float16,fp8,4095,0.44493333498636883
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,1,float16,float16,8191,1.0927786827087402
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,1,float16,fp8,8191,0.8782506783803304
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,1,float16,float16,1,0.012560000022252401
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,1,float16,fp8,1,0.01239466667175293
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,1,float16,float16,3,0.013471999516089758
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,1,float16,float16,7,0.013781332721312841
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,1,float16,fp8,3,0.01314666618903478
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,1,float16,fp8,7,0.014394666999578476
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,1,float16,fp8,15,0.012970666090647379
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,1,float16,float16,15,0.012709333250919977
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,1,float16,float16,31,0.01239466667175293
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,1,float16,fp8,31,0.012341332932313284
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,1,float16,float16,63,0.012890666723251343
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,1,float16,fp8,63,0.012719999998807907
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,1,float16,float16,127,0.013653332988421122
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,1,float16,fp8,127,0.012346666306257248
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,1,float16,float16,255,0.012554666648308435
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,1,float16,fp8,255,0.012576000144084295
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,1,float16,fp8,511,0.014416000495354334
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,1,float16,float16,511,0.01461333284775416
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,1,float16,float16,1023,0.016666666915019352
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,1,float16,fp8,1023,0.014511999984582266
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,1,float16,float16,2047,0.023103999594847362
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,1,float16,fp8,2047,0.02075733368595441
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,1,float16,float16,4095,0.03958400090535482
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,1,float16,fp8,4095,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,1,float16,float16,8191,0.060415998101234436
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,1,float16,fp8,8191,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,1,float16,fp8,1,0.024847999215126038
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,1,float16,float16,3,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,1,float16,fp8,3,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,1,float16,float16,7,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,1,float16,fp8,7,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,1,float16,float16,15,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,1,float16,fp8,15,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,1,float16,float16,31,0.03310399999221166
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,1,float16,fp8,31,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,1,float16,float16,63,0.034815999368826546
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,1,float16,fp8,63,0.03105599929889043
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,1,float16,float16,127,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,1,float16,fp8,127,0.03585600107908249
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,1,float16,float16,255,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,1,float16,fp8,255,0.053247998158137
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,1,float16,float16,511,0.09462400277455647
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,1,float16,fp8,511,0.08635733524958293
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,1,float16,float16,1023,0.16827734311421713
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,1,float16,fp8,1023,0.15411200126012167
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,1,float16,float16,2047,0.3131626645723979
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,1,float16,fp8,2047,0.29098665714263916
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,1,float16,float16,4095,0.6027946472167969
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,1,float16,fp8,4095,0.5618346532185873
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,1,float16,float16,8191,1.226746638615926
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,1,float16,fp8,8191,1.106101353963216
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,1,float16,float16,1,0.029018667836983997
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,1,float16,fp8,1,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,1,float16,float16,3,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,1,float16,fp8,3,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,1,float16,float16,7,0.030720000465710957
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,1,float16,fp8,7,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,1,float16,float16,15,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,1,float16,fp8,15,0.030720000465710957
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,1,float16,float16,31,0.034474665919939675
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,1,float16,fp8,31,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,1,float16,float16,127,0.040965333580970764
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,1,float16,fp8,127,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,1,float16,float16,63,0.034128000338872276
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,1,float16,fp8,63,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,1,float16,float16,255,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,1,float16,fp8,255,0.05221866567929586
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,1,float16,float16,511,0.09454933802286784
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,1,float16,fp8,511,0.08635733524958293
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,1,float16,float16,1023,0.16810667514801025
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,1,float16,fp8,1023,0.15411200126012167
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,1,float16,float16,2047,0.3131733338038127
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,1,float16,fp8,2047,0.2892799973487854
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,1,float16,float16,4095,0.6167946656545004
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,1,float16,fp8,4095,0.5614933172861735
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,1,float16,fp8,8191,1.1088213125864665
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,1,float16,float16,8191,1.3608959515889485
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,1,float16,float16,1,0.014783999572197596
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,1,float16,fp8,1,0.013034666577974955
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,1,float16,float16,3,0.012560000022252401
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,1,float16,fp8,3,0.014335999886194864
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,1,float16,float16,7,0.012981332838535309
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,1,float16,fp8,7,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,1,float16,float16,15,0.014432000617186228
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,1,float16,fp8,15,0.013045333325862885
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,1,float16,float16,31,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,1,float16,fp8,31,0.014032000054915747
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,1,float16,float16,63,0.013306666165590286
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,1,float16,fp8,63,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,1,float16,float16,127,0.013669333110253016
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,1,float16,fp8,127,0.01268799975514412
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,1,float16,float16,255,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,1,float16,fp8,255,0.013327999661366144
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,1,float16,fp8,511,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,1,float16,float16,1023,0.01666133354107539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,1,float16,fp8,1023,0.014677333335081736
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,1,float16,float16,2047,0.022805333137512207
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,1,float16,fp8,2047,0.02075733368595441
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,1,float16,float16,4095,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,1,float16,fp8,4095,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,1,float16,float16,8191,0.06109866499900818
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,1,float16,fp8,8191,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,1,float16,float16,1,0.014111999422311783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,1,float16,fp8,1,0.012901333471139273
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,1,float16,float16,3,0.014495999862750372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,1,float16,fp8,3,0.012906666845083237
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,1,float16,float16,7,0.015189333508412043
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,1,float16,fp8,7,0.0129120002190272
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,1,float16,float16,15,0.015295999745527903
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,1,float16,fp8,15,0.013082666943470636
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,1,float16,float16,31,0.0143306665122509
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,1,float16,fp8,31,0.012906666845083237
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,1,float16,float16,63,0.014463999619086584
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,1,float16,fp8,63,0.012901333471139273
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,1,float16,float16,127,0.013066666821638743
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,1,float16,fp8,127,0.012906666845083237
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,1,float16,float16,255,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,1,float16,fp8,255,0.01314666618903478
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,1,float16,float16,511,0.01655999943614006
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,1,float16,fp8,511,0.014789332946141561
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,1,float16,float16,1023,0.019039999693632126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,1,float16,fp8,1023,0.018719999740521114
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,1,float16,float16,2047,0.03994133323431015
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,1,float16,fp8,2047,0.02696000039577484
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,1,float16,float16,4095,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,1,float16,fp8,4095,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,1,float16,float16,8191,0.10410666465759277
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,1,float16,float16,1,0.01128000020980835
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,1,float16,fp8,8191,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,1,float16,fp8,1,0.010389333590865135
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,1,float16,float16,3,0.01033599985142549
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,1,float16,fp8,3,0.01110400011142095
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,1,float16,float16,7,0.010922666639089584
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,1,float16,float16,15,0.01121066634853681
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,1,float16,fp8,7,0.011285333583752314
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,1,float16,fp8,15,0.011312000453472137
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,1,float16,float16,31,0.011407999942700068
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,1,float16,fp8,31,0.011429333438475927
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,1,float16,float16,63,0.011429333438475927
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,1,float16,fp8,63,0.01156266654531161
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,1,float16,float16,127,0.011258666714032492
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,1,float16,fp8,127,0.01209066684047381
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,1,float16,float16,255,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,1,float16,fp8,255,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,1,float16,float16,511,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,1,float16,fp8,511,0.02252800017595291
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,1,float16,float16,1023,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,1,float16,fp8,1023,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,1,float16,float16,2047,0.036864000062147774
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,1,float16,fp8,2047,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,1,float16,float16,4095,0.05358933409055074
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,1,float16,fp8,4095,0.04949333270390829
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,1,float16,float16,8191,0.08841066559155782
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,1,float16,fp8,8191,0.08021333316961925
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,1,float16,float16,16383,0.15598932902018228
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,1,float16,fp8,16383,0.1397760013739268
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,1,float16,float16,32767,0.29201066493988037
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,1,float16,float16,1,0.010538666198650995
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,1,float16,fp8,32767,0.2624853253364563
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,1,float16,fp8,1,0.01051733394463857
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,1,float16,float16,3,0.009658666948477427
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,1,float16,float16,7,0.010304000228643417
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,1,float16,fp8,3,0.011424000064531961
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,1,float16,fp8,7,0.010037333394090334
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,1,float16,float16,15,0.010079999764760336
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,1,float16,fp8,15,0.011535999675591787
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,1,float16,float16,31,0.011231999844312668
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,1,float16,fp8,31,0.011776000261306763
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,1,float16,float16,63,0.011509332805871964
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,1,float16,fp8,63,0.011685332904259363
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,1,float16,float16,127,0.011594666788975397
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,1,float16,fp8,127,0.011535999675591787
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,1,float16,float16,255,0.0145066666106383
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,1,float16,fp8,255,0.014677333335081736
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,1,float16,float16,511,0.022863999009132385
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,1,float16,fp8,511,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,1,float16,float16,1023,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,1,float16,fp8,1023,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,1,float16,float16,2047,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,1,float16,fp8,2047,0.034117333590984344
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,1,float16,float16,4095,0.05358933409055074
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,1,float16,fp8,4095,0.049498667319615684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,1,float16,float16,8191,0.08840533097585042
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,1,float16,fp8,8191,0.08021333316961925
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,1,float16,float16,16383,0.15581867098808289
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,1,float16,fp8,16383,0.14148267110188803
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,1,float16,float16,32767,0.2926933368047078
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,1,float16,float16,1,0.012479999413092932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,1,float16,fp8,32767,0.26146666208902997
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,1,float16,fp8,1,0.013125333935022354
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,1,float16,float16,3,0.010703999549150467
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,1,float16,float16,7,0.011029332876205444
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,1,float16,fp8,7,0.010693332801262537
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,1,float16,float16,15,0.011274666835864386
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,1,float16,fp8,15,0.011029332876205444
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,1,float16,float16,31,0.011034666250149408
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,1,float16,fp8,31,0.011802667131026586
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,1,float16,float16,63,0.010863999525705973
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,1,float16,fp8,63,0.010853332777818045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,1,float16,float16,127,0.011882666498422623
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,1,float16,fp8,127,0.012085333466529846
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,1,float16,float16,255,0.011776000261306763
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,1,float16,fp8,255,0.012319999436537424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,1,float16,float16,511,0.011952000359694162
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,1,float16,fp8,511,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,1,float16,float16,1023,0.012389333297808966
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,1,float16,fp8,1023,0.012560000022252401
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,1,float16,float16,2047,0.012906666845083237
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,1,float16,fp8,2047,0.012901333471139273
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,1,float16,float16,4095,0.0145066666106383
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,1,float16,fp8,4095,0.013605333864688873
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,1,float16,float16,8191,0.02065066620707512
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,1,float16,fp8,8191,0.02082666630546252
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,1,float16,float16,16383,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,1,float16,fp8,16383,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,1,float16,float16,32767,0.025605333348115284
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,1,float16,fp8,32767,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,1,float16,float16,1,0.012383999923865
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,1,float16,fp8,1,0.012597333639860153
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,1,float16,float16,3,0.011957333733638128
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,1,float16,fp8,3,0.013482666263977686
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,1,float16,float16,7,0.012608000387748083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,1,float16,fp8,7,0.010869332899649939
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,1,float16,float16,15,0.011370666325092316
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,1,float16,fp8,15,0.010863999525705973
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,1,float16,float16,31,0.01109333336353302
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,1,float16,fp8,31,0.0122079998254776
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,1,float16,float16,63,0.011610666910807291
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,1,float16,fp8,63,0.011503999431928
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,1,float16,float16,127,0.011039999624093374
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,1,float16,fp8,127,0.012047999848922094
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,1,float16,float16,255,0.011770666887362799
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,1,float16,fp8,255,0.011861333002646765
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,1,float16,float16,511,0.011968000481526056
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,1,float16,fp8,511,0.012970666090647379
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,1,float16,float16,1023,0.012389333297808966
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,1,float16,fp8,1023,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,1,float16,float16,2047,0.0129120002190272
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,1,float16,fp8,2047,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,1,float16,float16,4095,0.016666666915019352
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,1,float16,fp8,4095,0.01598400001724561
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,1,float16,float16,1,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,1,float16,float16,8191,0.01870399961868922
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,1,float16,fp8,8191,0.0186666672428449
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,1,float16,float16,16383,0.020768000433842342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,1,float16,fp8,16383,0.020714666694402695
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,1,float16,float16,32767,0.022800001005331676
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,1,float16,fp8,32767,0.022757334013779957
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,1,float16,float16,1,0.011285333583752314
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,1,float16,fp8,1,0.011866666376590729
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,1,float16,float16,3,0.01081066702802976
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,1,float16,fp8,3,0.01137599969903628
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,1,float16,float16,7,0.011296000331640244
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,1,float16,fp8,7,0.012256000190973282
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,1,float16,float16,15,0.012053333222866058
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,1,float16,fp8,15,0.011695999652147293
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,1,float16,float16,31,0.01156266654531161
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,1,float16,fp8,31,0.012144000579913458
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,1,float16,float16,63,0.011183999478816986
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,1,float16,fp8,63,0.012261333564917246
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,1,float16,float16,127,0.01180800050497055
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,1,float16,fp8,127,0.012234666695197424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,1,float16,float16,255,0.022810667753219604
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,1,float16,fp8,255,0.02275199939807256
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,1,float16,float16,511,0.024847999215126038
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,1,float16,fp8,511,0.025098666548728943
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,1,float16,float16,1023,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,1,float16,fp8,1023,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,1,float16,float16,2047,0.04539733131726583
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,1,float16,fp8,2047,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,1,float16,float16,4095,0.07202666501204173
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,1,float16,fp8,4095,0.06587733328342438
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,1,float16,float16,8191,0.12321600317955017
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,1,float16,fp8,8191,0.11264000336329143
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,1,float16,float16,16383,0.2259626587231954
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,1,float16,fp8,16383,0.2032639980316162
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,1,float16,float16,32767,0.43059198061625165
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,1,float16,fp8,32767,0.3858826557795207
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,1,float16,float16,1,0.010037333394090334
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,1,float16,fp8,1,0.012954667210578918
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,1,float16,float16,3,0.014090667168299357
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,1,float16,fp8,3,0.011621333658695221
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,1,float16,float16,7,0.010330666477481524
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,1,float16,fp8,7,0.011850666254758835
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,1,float16,float16,15,0.011616000284751257
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,1,float16,fp8,15,0.011648000528415045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,1,float16,float16,31,0.01138666644692421
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,1,float16,fp8,31,0.012138667205969492
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,1,float16,float16,63,0.011994666109482447
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,1,float16,fp8,63,0.012063999970753988
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,1,float16,float16,127,0.011999999483426413
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,1,float16,fp8,127,0.011829332758982977
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,1,float16,float16,255,0.022810667753219604
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,1,float16,fp8,255,0.02276800076166789
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,1,float16,float16,511,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,1,float16,fp8,511,0.024911999702453613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,1,float16,float16,1023,0.03310399999221166
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,1,float16,fp8,1023,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,1,float16,float16,2047,0.04573333263397217
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,1,float16,fp8,2047,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,1,float16,float16,4095,0.07202133536338806
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,1,float16,fp8,4095,0.06587733328342438
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,1,float16,float16,8191,0.12322133779525757
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,1,float16,fp8,8191,0.1129813293615977
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,1,float16,float16,16383,0.22562134265899658
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,1,float16,fp8,16383,0.20514132579167685
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,1,float16,float16,32767,0.4312746524810791
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,1,float16,fp8,32767,0.385535995165507
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,1,float16,float16,1,0.012346666306257248
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,1,float16,fp8,1,0.014074667046467463
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,1,float16,float16,3,0.010863999525705973
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,1,float16,fp8,3,0.01349866638580958
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,1,float16,float16,7,0.012896000097195307
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,1,float16,fp8,7,0.013898666948080063
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,1,float16,float16,15,0.012437333663304647
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,1,float16,fp8,15,0.013141332815090815
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,1,float16,float16,31,0.012458667159080505
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,1,float16,fp8,31,0.013536000003417334
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,1,float16,float16,63,0.012453333785136541
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,1,float16,fp8,63,0.01331199953953425
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,1,float16,float16,127,0.012890666723251343
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,1,float16,fp8,127,0.013823999712864557
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,1,float16,float16,255,0.012752000242471695
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,1,float16,fp8,255,0.013530666629473368
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,1,float16,float16,511,0.01239466667175293
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,1,float16,fp8,511,0.013477332890033722
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,1,float16,float16,1023,0.015872000406185787
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,1,float16,fp8,1023,0.013306666165590286
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,1,float16,float16,2047,0.012917333592971167
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,1,float16,fp8,2047,0.014570667097965876
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,1,float16,float16,4095,0.017685333887736004
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,1,float16,fp8,4095,0.0186666672428449
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,1,float16,float16,8191,0.02082666630546252
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,1,float16,fp8,8191,0.020725333442290623
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,1,float16,float16,16383,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,1,float16,float16,511,0.014432000617186228
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,1,float16,fp8,16383,0.022863999009132385
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,1,float16,float16,32767,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,1,float16,fp8,32767,0.02481066683928172
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,1,float16,float16,1,0.012538666526476542
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,1,float16,fp8,1,0.012144000579913458
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,1,float16,float16,3,0.012389333297808966
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,1,float16,fp8,3,0.012608000387748083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,1,float16,float16,7,0.011749333391586939
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,1,float16,fp8,7,0.011882666498422623
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,1,float16,float16,15,0.013999999811251959
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,1,float16,fp8,15,0.012821332861979803
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,1,float16,float16,31,0.012170666207869848
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,1,float16,fp8,31,0.013093333691358566
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,1,float16,float16,63,0.01202133297920227
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,1,float16,fp8,63,0.013754667093356451
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,1,float16,float16,127,0.010981333752473196
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,1,float16,fp8,127,0.011877333124478659
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,1,float16,float16,255,0.013045333325862885
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,1,float16,fp8,255,0.012554666648308435
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,1,float16,float16,511,0.012560000022252401
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,1,float16,fp8,511,0.012357333054145178
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,1,float16,fp8,1023,0.012741333494583765
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,1,float16,float16,1023,0.012965332716703415
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,1,float16,float16,2047,0.014890667051076889
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,1,float16,fp8,2047,0.016650666793187458
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,1,float16,float16,4095,0.01729600007335345
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,1,float16,fp8,4095,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,1,float16,float16,8191,0.02075200031201045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,1,float16,fp8,8191,0.018863999595244724
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,1,float16,float16,16383,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,1,float16,fp8,16383,0.02075200031201045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,1,float16,float16,32767,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,1,float16,fp8,32767,0.02628266563018163
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,1,float16,float16,1,0.049498667319615684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,1,float16,fp8,1,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,1,float16,float16,3,0.051541333397229515
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,1,float16,fp8,3,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,1,float16,float16,7,0.0517439991235733
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,1,float16,fp8,7,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,1,float16,float16,15,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,1,float16,fp8,15,0.05358933409055074
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,1,float16,float16,31,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,1,float16,fp8,31,0.05358933409055074
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,1,float16,float16,63,0.0628053347269694
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,1,float16,fp8,63,0.05358933409055074
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,1,float16,float16,127,0.07406933108965556
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,1,float16,fp8,127,0.06382933259010315
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,1,float16,float16,255,0.1088853379090627
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,1,float16,fp8,255,0.09659733374913533
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,1,float16,float16,511,0.17937066157658896
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,1,float16,fp8,511,0.1599146624406179
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,1,float16,float16,1023,0.3176106611887614
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,1,float16,fp8,1023,0.29098665714263916
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,1,float16,float16,2047,0.5925493240356445
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,1,float16,fp8,2047,0.5474986632664999
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,1,float16,float16,4095,1.196031967798869
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,1,float16,float16,1,0.04949333270390829
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,1,float16,fp8,4095,1.0661546389261882
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,1,float16,fp8,1,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,1,float16,float16,3,0.05154666801293691
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,1,float16,fp8,3,0.04164266586303711
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,1,float16,float16,7,0.05152533451716105
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,1,float16,fp8,7,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,1,float16,float16,15,0.06177600224812826
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,1,float16,fp8,15,0.05555733541647593
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,1,float16,float16,31,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,1,float16,fp8,31,0.05359466870625814
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,1,float16,float16,63,0.06382933259010315
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,1,float16,fp8,63,0.05358933409055074
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,1,float16,float16,127,0.07406933108965556
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,1,float16,fp8,127,0.06485333542029063
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,1,float16,float16,255,0.1088853379090627
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,1,float16,fp8,255,0.09659733374913533
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,1,float16,float16,511,0.17834667364756265
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,1,float16,fp8,511,0.16008533040682474
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,1,float16,float16,1023,0.31590400139490765
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,1,float16,fp8,1023,0.2913279930750529
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,1,float16,float16,2047,0.6297599871953329
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,1,float16,fp8,2047,0.5471573273340861
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,1,float16,float16,4095,1.2767573197682698
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,1,float16,float16,1,0.014906667172908783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,1,float16,fp8,4095,1.0692213376363118
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,1,float16,fp8,1,0.014912000546852747
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,1,float16,float16,3,0.014954666296641031
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,1,float16,fp8,3,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,1,float16,float16,7,0.014906667172908783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,1,float16,fp8,7,0.015072000523408255
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,1,float16,float16,15,0.014949332922697067
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,1,float16,fp8,15,0.015082667271296183
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,1,float16,float16,31,0.015077333897352219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,1,float16,fp8,31,0.01443733274936676
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,1,float16,float16,63,0.015253332753976187
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,1,float16,fp8,63,0.014906667172908783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,1,float16,float16,127,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,1,float16,fp8,127,0.014581333845853806
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,1,float16,float16,255,0.015087999403476715
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,1,float16,fp8,255,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,1,float16,float16,511,0.018863999595244724
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,1,float16,fp8,511,0.016783999900023144
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,1,float16,float16,1023,0.022122666239738464
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,1,float16,fp8,1023,0.020703999946514767
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,1,float16,float16,2047,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,1,float16,fp8,2047,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,1,float16,float16,4095,0.06280000011126201
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,1,float16,fp8,4095,0.04539733131726583
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,1,float16,float16,1,0.016666666915019352
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,1,float16,fp8,1,0.016623999923467636
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,1,float16,float16,3,0.016842667013406754
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,1,float16,fp8,3,0.016656000167131424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,1,float16,float16,7,0.0170666662355264
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,1,float16,fp8,7,0.01525866612792015
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,1,float16,float16,15,0.016901332885026932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,1,float16,fp8,15,0.016602666427691776
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,1,float16,float16,31,0.016976000120242436
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,1,float16,fp8,31,0.01461333284775416
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,1,float16,float16,63,0.016970666746298473
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,1,float16,fp8,63,0.016677333662907284
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,1,float16,float16,127,0.016650666793187458
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,1,float16,fp8,127,0.016447999825080235
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,1,float16,float16,255,0.017018667111794155
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,1,float16,fp8,255,0.016666666915019352
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,1,float16,float16,511,0.0210506667693456
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,1,float16,fp8,511,0.018858666221300762
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,1,float16,float16,1023,0.03721066564321518
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,1,float16,fp8,1023,0.024821333587169647
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,1,float16,float16,2047,0.062122667829195656
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,1,float16,fp8,2047,0.04164266586303711
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,1,float16,float16,4095,0.10444800059000652
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,1,float16,float16,1,0.014783999572197596
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,1,float16,fp8,4095,0.06383466720581055
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,1,float16,fp8,1,0.012298667182525
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,1,float16,float16,3,0.014458666245142618
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,1,float16,fp8,3,0.01357866699496905
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,1,float16,float16,7,0.014741333822409311
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,1,float16,fp8,3,0.011600000162919363
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,1,float16,fp8,7,0.011567999919255575
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,1,float16,float16,15,0.014741333822409311
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,1,float16,fp8,15,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,1,float16,float16,31,0.016490666816631954
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,1,float16,fp8,31,0.014293332894643148
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,1,float16,float16,63,0.01672533278663953
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,1,float16,fp8,63,0.014570667097965876
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,1,float16,float16,127,0.01666133354107539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,1,float16,fp8,127,0.013631999492645264
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,1,float16,float16,255,0.018768000106016796
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,1,float16,fp8,255,0.014576000471909841
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,1,float16,float16,511,0.024842667082945507
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,1,float16,fp8,511,0.020645332833131153
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,1,float16,float16,1023,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,1,float16,fp8,1023,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,1,float16,float16,2047,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,1,float16,fp8,2047,0.04948799808820089
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,1,float16,float16,4095,0.1088800032933553
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,1,float16,fp8,4095,0.0897706647713979
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,1,float16,float16,8191,0.20514132579167685
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,1,float16,fp8,8191,0.16606400410334268
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,1,float16,float16,16383,0.39695998032887775
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,1,float16,fp8,16383,0.32546667257944745
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,1,float16,float16,32767,0.7819946606953939
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,1,float16,float16,1,0.014906667172908783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,1,float16,fp8,1,0.011578666667143503
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,1,float16,float16,3,0.016501333564519882
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,1,float16,fp8,3,0.01190399999419848
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,1,float16,float16,7,0.015013333410024643
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,1,float16,fp8,7,0.01180800050497055
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,1,float16,float16,15,0.014576000471909841
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,1,float16,fp8,15,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,1,float16,float16,31,0.016149333367745083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,1,float16,fp8,31,0.01470400020480156
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,1,float16,float16,63,0.01661866654952367
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,1,float16,fp8,63,0.012458667159080505
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,1,float16,float16,127,0.016623999923467636
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,1,float16,fp8,127,0.013823999712864557
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,1,float16,float16,255,0.01887999971707662
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,1,float16,fp8,255,0.014576000471909841
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,1,float16,float16,511,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,1,float16,fp8,511,0.019189332922299702
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,1,float16,float16,1023,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,1,float16,fp8,1023,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,1,float16,float16,2047,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,1,float16,fp8,2047,0.049498667319615684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,1,float16,float16,4095,0.1088853379090627
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,1,float16,fp8,4095,0.08840533097585042
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,1,float16,float16,8191,0.20497065782546997
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,1,float16,fp8,8191,0.16861865917841592
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,1,float16,float16,16383,0.3973120053609212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,1,float16,fp8,16383,0.32204800844192505
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,1,float16,float16,32767,0.8166506290435791
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,1,float16,float16,1,0.011685332904259363
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,1,float16,fp8,32767,0.6775466601053873
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,1,float16,fp8,1,0.012960000584522883
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,1,float16,float16,3,0.01231466606259346
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,1,float16,fp8,3,0.01379199946920077
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,1,float16,float16,7,0.012186666329701742
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,1,float16,fp8,7,0.013525333255529404
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,1,float16,float16,15,0.01232533281048139
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,1,float16,fp8,15,0.013141332815090815
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,1,float16,float16,31,0.012181332955757776
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,1,float16,float16,63,0.013477332890033722
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,1,float16,fp8,31,0.012319999436537424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,1,float16,fp8,63,0.012063999970753988
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,1,float16,float16,127,0.01370666672786077
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,1,float16,fp8,127,0.01251199965675672
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,1,float16,float16,255,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,1,float16,fp8,255,0.01268799975514412
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,1,float16,float16,511,0.01349866638580958
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,1,float16,fp8,511,0.013301332791646322
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,1,float16,float16,1023,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,1,float16,fp8,1023,0.014357333381970724
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,1,float16,float16,2047,0.018677332748969395
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,1,float16,fp8,2047,0.01871466636657715
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,1,float16,float16,4095,0.01894933357834816
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,1,float16,fp8,4095,0.019002666076024372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,1,float16,float16,8191,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,1,float16,fp8,8191,0.021013334393501282
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,1,float16,float16,16383,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,1,float16,fp8,16383,0.024911999702453613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,1,float16,float16,32767,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,1,float16,float16,1,0.012789333860079447
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,1,float16,fp8,32767,0.02902399996916453
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,1,float16,fp8,1,0.012650666137536367
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,1,float16,float16,3,0.010821333775917688
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,1,float16,fp8,3,0.013584000368913015
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,1,float16,float16,7,0.012975999464591345
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,1,float16,fp8,7,0.014479999740918478
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,1,float16,float16,15,0.012261333564917246
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,1,float16,fp8,15,0.012261333564917246
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,1,float16,float16,31,0.012410666793584824
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,1,float16,fp8,31,0.01394133393963178
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,1,float16,float16,63,0.012437333663304647
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,1,float16,fp8,63,0.012746666868527731
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,1,float16,float16,127,0.012725333372751871
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,1,float16,fp8,127,0.01349866638580958
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,1,float16,float16,255,0.01232533281048139
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,1,float16,fp8,255,0.011157333850860596
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,1,float16,float16,511,0.01370666672786077
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,1,float16,fp8,511,0.014010666559139887
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,1,float16,float16,1023,0.012629333883523941
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,1,float16,fp8,1023,0.014848000059525171
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,1,float16,float16,2047,0.016623999923467636
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,1,float16,fp8,2047,0.014767999450365702
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,1,float16,float16,4095,0.018709332992633183
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,1,float16,fp8,4095,0.016623999923467636
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,1,float16,float16,8191,0.021370666722456615
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,1,float16,fp8,8191,0.018709332992633183
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,1,float16,float16,16383,0.038906666139761605
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,1,float16,fp8,16383,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,1,float16,float16,32767,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,1,float16,float16,1,0.09454933802286784
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,1,float16,fp8,1,0.07611733178297679
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,1,float16,float16,3,0.09660266836484273
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,1,float16,fp8,3,0.07816533247629802
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,1,float16,float16,7,0.09864532947540283
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,1,float16,fp8,7,0.08191466828187306
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,1,float16,float16,15,0.11707733074824016
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,1,float16,fp8,15,0.10240000486373901
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,1,float16,float16,31,0.11707733074824016
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,1,float16,fp8,31,0.10240000486373901
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,1,float16,float16,63,0.11912533640861511
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,1,float16,fp8,63,0.10274666547775269
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,1,float16,float16,127,0.14097066720326742
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,1,float16,fp8,127,0.12117333213488261
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,1,float16,float16,255,0.20907199382781982
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,1,float16,fp8,255,0.18466132879257202
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,1,float16,float16,511,0.3466240167617798
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,1,float16,fp8,511,0.3118079900741577
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,1,float16,float16,1023,0.6208853324254354
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,1,float16,fp8,1023,0.570026675860087
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,1,float16,float16,2047,1.2093493143717449
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,1,float16,fp8,2047,1.0876586437225342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,1,float16,float16,1,0.09489066402117412
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,1,float16,fp8,1,0.07611733178297679
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,1,float16,float16,3,0.09730666875839233
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,1,float16,fp8,3,0.07816533247629802
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,1,float16,float16,7,0.0986400047938029
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,1,float16,fp8,7,0.08226133386294048
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,1,float16,float16,15,0.11707733074824016
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,1,float16,fp8,15,0.10069333513577779
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,1,float16,float16,31,0.11707733074824016
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,1,float16,fp8,31,0.10274133086204529
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,1,float16,float16,63,0.1181013286113739
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,1,float16,fp8,63,0.10274133086204529
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,1,float16,float16,127,0.14148267110188803
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,1,float16,fp8,127,0.12117333213488261
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,1,float16,float16,255,0.20923733711242676
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,1,float16,fp8,255,0.18449600537618002
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,1,float16,float16,511,0.3466240167617798
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,1,float16,fp8,511,0.3114666740099589
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,1,float16,float16,1023,0.6476800044377645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,1,float16,fp8,1023,0.5679786602656046
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,1,float16,float16,2047,1.2682240009307861
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,1,float16,fp8,2047,1.0910720030466716
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,1,float16,float16,1,0.020714666694402695
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,1,float16,fp8,1,0.01871466636657715
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,1,float16,float16,3,0.02073066681623459
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,1,float16,fp8,3,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,1,float16,float16,7,0.020581333587567013
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,1,float16,fp8,7,0.01870399961868922
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,1,float16,float16,15,0.019002666076024372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,1,float16,fp8,15,0.01870399961868922
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,1,float16,float16,31,0.020714666694402695
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,1,float16,fp8,31,0.018672000616788864
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,1,float16,float16,63,0.02070933332045873
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,1,float16,fp8,63,0.018698666244745255
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,1,float16,float16,127,0.020714666694402695
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,1,float16,fp8,127,0.01870399961868922
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,1,float16,float16,255,0.020703999946514767
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,1,float16,fp8,255,0.01870399961868922
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,1,float16,float16,511,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,1,float16,fp8,511,0.02219199885924657
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,1,float16,fp8,1023,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,1,float16,float16,2047,0.06384000182151794
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,1,float16,fp8,2047,0.0460746685663859
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,1,float16,float16,1,0.022805333137512207
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,1,float16,fp8,1,0.020746666938066483
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,1,float16,float16,3,0.022810667753219604
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,1,float16,fp8,3,0.02075733368595441
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,1,float16,float16,7,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,1,float16,fp8,7,0.020746666938066483
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,1,float16,float16,15,0.022800001005331676
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,1,float16,fp8,15,0.020655999581019085
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,1,float16,float16,31,0.022810667753219604
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,1,float16,fp8,31,0.02075733368595441
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,1,float16,float16,63,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,1,float16,fp8,63,0.01974933346112569
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,1,float16,float16,127,0.022800001005331676
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,1,float16,fp8,127,0.02075200031201045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,1,float16,float16,255,0.022800001005331676
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,1,float16,fp8,255,0.020762667059898376
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,1,float16,float16,511,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,1,float16,fp8,511,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,1,float16,float16,1023,0.05563733478387197
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,1,float16,fp8,1023,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,1,float16,float16,2047,0.09864532947540283
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,1,float16,fp8,2047,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,1,float16,float16,1,0.18449066082636514
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,1,float16,fp8,1,0.14574933052062988
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,1,float16,float16,3,0.18858667214711508
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,1,float16,fp8,3,0.14967466394106546
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,1,float16,float16,7,0.19114132722218832
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,1,float16,fp8,7,0.15684266885121664
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,1,float16,float16,15,0.22732800245285034
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,1,float16,fp8,15,0.19694934288660684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,1,float16,float16,31,0.22971733411153158
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,1,float16,fp8,31,0.19712533553441366
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,1,float16,float16,63,0.22971733411153158
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,1,float16,fp8,63,0.19882667064666748
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,1,float16,float16,127,0.27426133553187054
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,1,float16,fp8,127,0.2355253299077352
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,1,float16,float16,255,0.4089173475901286
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,1,float16,fp8,255,0.3609600067138672
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,1,float16,float16,511,0.6840319633483887
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,1,float16,fp8,511,0.6140586535135905
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,1,float16,float16,1,0.18449066082636514
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,1,float16,fp8,1,0.14574399590492249
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,1,float16,float16,3,0.18773333231608072
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,1,float16,fp8,3,0.15001599987347922
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,1,float16,float16,7,0.1909760038057963
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,1,float16,fp8,7,0.15633066495259604
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,1,float16,float16,15,0.22733332713445029
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,1,float16,fp8,15,0.1971199909845988
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,1,float16,float16,31,0.22937599817911783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,1,float16,fp8,31,0.19694934288660684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,1,float16,float16,63,0.230730672677358
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,1,float16,fp8,63,0.19883199532826742
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,1,float16,float16,127,0.27459200223286945
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,1,float16,fp8,127,0.23381332556406656
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,1,float16,float16,255,0.40958933035532635
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,1,float16,fp8,255,0.3609600067138672
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,1,float16,float16,511,0.7219200134277344
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,1,float16,fp8,511,0.6137119928995768
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,1,float16,float16,1,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,1,float16,fp8,1,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,1,float16,float16,3,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,1,float16,fp8,3,0.026848000784715016
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,1,float16,float16,7,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,1,float16,fp8,7,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,1,float16,float16,15,0.029018667836983997
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,1,float16,fp8,15,0.026848000784715016
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,1,float16,float16,31,0.029018667836983997
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,1,float16,fp8,31,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,1,float16,float16,63,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,1,float16,fp8,63,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,1,float16,float16,127,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,1,float16,fp8,32767,0.6768799622853597
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,1,float16,fp8,127,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,1,float16,float16,255,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,1,float16,fp8,255,0.024858665963013966
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,1,float16,float16,511,0.03925866633653641
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,1,float16,fp8,511,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,1,float16,float16,1,0.03719999889532725
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,1,float16,fp8,1,0.02902399996916453
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,1,float16,float16,3,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,1,float16,fp8,3,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,1,float16,float16,7,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,1,float16,fp8,7,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,1,float16,float16,15,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,1,float16,fp8,15,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,1,float16,float16,31,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,1,float16,fp8,31,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,1,float16,float16,63,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,1,float16,fp8,63,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,1,float16,float16,127,0.03585066646337509
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,1,float16,fp8,127,0.030720000465710957
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,1,float16,float16,255,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,1,float16,fp8,255,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,1,float16,float16,511,0.058362667759259544
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,1,float16,fp8,511,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,1,float16,float16,1,0.3609600067138672
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,1,float16,fp8,1,0.28279467423756915
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,1,float16,float16,3,0.3691519896189372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,1,float16,fp8,3,0.2930346727371216
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,1,float16,float16,7,0.37563733259836835
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,1,float16,fp8,7,0.304639995098114
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,1,float16,float16,15,0.44697598616282147
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,1,float16,fp8,15,0.38758933544158936
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,1,float16,float16,31,0.4514133135477702
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,1,float16,float16,63,0.45482667287190753
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,1,float16,fp8,31,0.3872426748275757
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,1,float16,fp8,63,0.3896266619364421
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,1,float16,float16,127,0.5416959921518961
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,1,float16,fp8,127,0.4609760046005249
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,1,float16,float16,255,0.8094666798909506
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,1,float16,fp8,255,0.7144052982330322
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,1,float16,float16,1,0.36061867078145343
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,1,float16,float16,3,0.3688160181045532
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,1,float16,fp8,1,0.28279467423756915
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,1,float16,fp8,3,0.2923520008722941
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,1,float16,float16,7,0.3752959966659546
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,1,float16,fp8,7,0.3056640028953552
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,1,float16,float16,15,0.4469706614812215
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,1,float16,fp8,15,0.3872426748275757
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,1,float16,float16,31,0.4517546494801839
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,1,float16,fp8,31,0.38758401075998944
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,1,float16,float16,63,0.45482667287190753
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,1,float16,fp8,63,0.3892800013224284
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,1,float16,float16,127,0.5474986632664999
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,1,float16,fp8,32767,0.039594667653242745
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,1,float16,fp8,127,0.46131734053293866
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,1,float16,float16,255,0.8622026443481445
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,1,float16,fp8,255,0.7181759675343832
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,1,float16,float16,1,0.04710400104522705
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,1,float16,fp8,1,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,1,float16,float16,3,0.04744533201058706
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,1,float16,fp8,3,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,1,float16,float16,7,0.04710400104522705
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,1,float16,fp8,7,0.04403733213742574
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,1,float16,float16,15,0.04539733131726583
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,1,float16,fp8,15,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,1,float16,float16,31,0.047450666626294456
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,1,float16,fp8,31,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,1,float16,float16,63,0.04710400104522705
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,1,float16,fp8,63,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,1,float16,float16,127,0.04744533201058706
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,1,float16,fp8,127,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,1,float16,float16,255,0.04949333270390829
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,1,float16,fp8,255,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,1,float16,float16,1,0.06382933259010315
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,1,float16,fp8,1,0.04983466863632202
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,1,float16,float16,3,0.06348800162474315
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,1,float16,fp8,3,0.04949333270390829
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,1,float16,float16,7,0.06313066681226094
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,1,float16,fp8,7,0.04949333270390829
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,1,float16,float16,15,0.06382933259010315
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,1,float16,fp8,15,0.04949333270390829
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,1,float16,float16,31,0.06382933259010315
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,1,float16,fp8,31,0.04949333270390829
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,1,float16,float16,63,0.06382933259010315
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,1,float16,fp8,63,0.04948799808820089
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,1,float16,float16,127,0.06382933259010315
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,1,float16,fp8,127,0.04982399940490723
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,1,float16,float16,255,0.06587733328342438
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,1,float16,fp8,255,0.05188799897829691
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,1,float16,float16,1,0.014394666999578476
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,1,float16,fp8,1,0.015306666493415833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,1,float16,float16,3,0.014576000471909841
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,1,float16,fp8,3,0.01661866654952367
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,1,float16,float16,7,0.014607999473810196
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,1,float16,fp8,7,0.014954666296641031
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,1,float16,float16,15,0.015696000307798386
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,1,float16,fp8,15,0.01672533278663953
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,1,float16,float16,31,0.014607999473810196
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,1,float16,fp8,31,0.020901332298914593
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,1,float16,float16,63,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,1,float16,fp8,63,0.02075200031201045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,1,float16,float16,127,0.016794666647911072
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,1,float16,fp8,127,0.02075200031201045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,1,float16,float16,255,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,1,float16,fp8,255,0.023103999594847362
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,1,float16,float16,511,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,1,float16,fp8,511,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,1,float16,float16,1023,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,1,float16,fp8,1023,0.05358933409055074
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,1,float16,float16,2047,0.10683733224868774
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,1,float16,fp8,2047,0.09081066648165385
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,1,float16,float16,4095,0.2010506590207418
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,1,float16,fp8,4095,0.16674133141835532
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,1,float16,float16,8191,0.39352532227834064
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,1,float16,fp8,8191,0.31965865691502887
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,1,float16,float16,16383,0.7866133054097494
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,1,float16,fp8,16383,0.6260053316752116
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,1,float16,float16,32767,2.69926389058431
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,1,float16,fp8,32767,1.3438293139139812
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,1,float16,float16,1,0.014741333822409311
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,1,float16,fp8,1,0.015466666469971338
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,1,float16,float16,3,0.014458666245142618
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,1,float16,fp8,3,0.016666666915019352
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,1,float16,float16,7,0.014490666488806406
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,1,float16,fp8,7,0.016650666793187458
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,1,float16,float16,15,0.014917333920796713
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,1,float16,fp8,15,0.01666133354107539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,1,float16,float16,31,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,1,float16,fp8,31,0.02070933332045873
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,1,float16,float16,63,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,1,float16,fp8,63,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,1,float16,float16,127,0.016970666746298473
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,1,float16,fp8,127,0.02065066620707512
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,1,float16,float16,255,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,1,float16,fp8,255,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,1,float16,float16,1023,0.038575999438762665
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,1,float16,float16,511,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,1,float16,fp8,511,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,1,float16,float16,1023,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,1,float16,float16,2047,0.10683733224868774
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,1,float16,fp8,2047,0.09045333663622539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,1,float16,fp8,1023,0.0525546669960022
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,1,float16,float16,4095,0.20481600364049277
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,1,float16,fp8,4095,0.16621866822242737
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,1,float16,float16,8191,0.3983360131581624
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,1,float16,fp8,8191,0.31965865691502887
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,1,float16,float16,16383,0.8166293303171793
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,1,float16,fp8,16383,0.6253226598103842
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,1,float16,fp8,32767,1.3608959515889485
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,1,float16,float16,32767,3.6034558614095054
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,1,float16,float16,1,0.011898666620254517
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,1,float16,fp8,1,0.012810666114091873
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,1,float16,float16,3,0.01331199953953425
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,1,float16,fp8,3,0.012202666451533636
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,1,float16,float16,7,0.014752000570297241
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,1,float16,fp8,7,0.012346666306257248
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,1,float16,float16,15,0.012416000167528788
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,1,float16,fp8,15,0.012223999947309494
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,1,float16,float16,31,0.01369599997997284
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,1,float16,fp8,31,0.01232533281048139
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,1,float16,float16,63,0.013855999956528345
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,1,float16,fp8,63,0.012335999558369318
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,1,float16,float16,127,0.013872000078360239
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,1,float16,fp8,127,0.012762666990359625
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,1,float16,float16,255,0.011968000481526056
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,1,float16,fp8,255,0.012624000509579977
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,1,float16,float16,511,0.01443733274936676
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,1,float16,fp8,511,0.014592000593741735
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,1,float16,float16,1023,0.014053333550691605
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,1,float16,fp8,1023,0.014933332800865173
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,1,float16,float16,2047,0.016965333372354507
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,1,float16,fp8,2047,0.01666133354107539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,1,float16,float16,4095,0.019007999449968338
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,1,float16,fp8,4095,0.01871466636657715
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,1,float16,float16,8191,0.022757334013779957
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,1,float16,fp8,8191,0.020879998803138733
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,1,float16,float16,16383,0.03958933303753535
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,1,float16,fp8,16383,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,1,float16,float16,32767,0.06007466713587443
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,1,float16,fp8,32767,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,1,float16,float16,1,0.012085333466529846
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,1,float16,fp8,1,0.013552000125249227
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,1,float16,float16,3,0.012351999680201212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,1,float16,fp8,3,0.0129120002190272
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,1,float16,float16,7,0.01221866657336553
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,1,float16,fp8,7,0.01210133358836174
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,1,float16,float16,15,0.013621332744757334
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,1,float16,fp8,15,0.012549333274364471
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,1,float16,float16,31,0.011600000162919363
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,1,float16,fp8,31,0.011978667229413986
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,1,float16,float16,63,0.012282667060693106
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,1,float16,fp8,63,0.012389333297808966
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,1,float16,float16,127,0.011893333246310553
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,1,float16,fp8,127,0.012442667037248611
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,1,float16,float16,255,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,1,float16,fp8,255,0.013605333864688873
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,1,float16,float16,511,0.012693333129088083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,1,float16,fp8,511,0.01313599944114685
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,1,float16,float16,1023,0.01422400027513504
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,1,float16,fp8,1023,0.014757333944241205
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,1,float16,float16,2047,0.018672000616788864
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,1,float16,fp8,2047,0.01664000004529953
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,1,float16,float16,4095,0.02184533327817917
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,1,float16,fp8,4095,0.02075733368595441
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,1,float16,float16,8191,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,1,float16,fp8,8191,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,1,float16,float16,16383,0.05938666562239329
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,1,float16,fp8,16383,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,1,float16,float16,32767,0.10069333513577779
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,1,float16,fp8,32767,0.05939733485380808
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,1,float16,float16,1,0.7137280305226644
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,1,float16,fp8,1,0.5587626695632935
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,1,float16,float16,3,0.7307039896647135
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,1,float16,fp8,3,0.579589327176412
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,1,float16,float16,7,0.7458133697509766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,1,float16,fp8,7,0.6017706791559855
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,1,float16,float16,15,0.8878080050150553
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,1,float16,fp8,15,0.76800537109375
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,1,float16,float16,31,0.899072011311849
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,1,float16,fp8,31,0.7683413028717041
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,1,float16,float16,63,0.90283203125
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,1,float16,fp8,63,0.772437334060669
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,1,float16,float16,127,1.0955146948496501
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,1,float16,float16,1,0.7127040227254232
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,1,float16,fp8,1,0.5577386617660522
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,1,float16,fp8,127,0.9154559771219889
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,1,float16,fp8,3,0.5775359869003296
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,1,float16,float16,3,0.733184019724528
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,1,float16,float16,7,0.7447840372721354
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,1,float16,fp8,7,0.6027946472167969
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,1,float16,float16,15,0.8888320128122965
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,1,float16,fp8,15,0.7714133262634277
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,1,float16,float16,31,0.8980426788330078
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,1,float16,fp8,31,0.7683413028717041
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,1,float16,fp8,63,0.7751680215199789
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,1,float16,float16,63,0.9171626567840576
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,1,float16,float16,127,1.1586559613545735
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,1,float16,fp8,127,0.9231359958648682
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,1,float16,float16,1,0.08566932876904805
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,1,float16,fp8,1,0.07815999786059062
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,1,float16,float16,3,0.0846506655216217
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,1,float16,fp8,3,0.077824001510938
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,1,float16,float16,7,0.08533333738644917
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,1,float16,fp8,7,0.07816533247629802
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,1,float16,float16,15,0.08533866206804912
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,1,float16,fp8,15,0.07816533247629802
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,1,float16,float16,31,0.08635200063387553
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,1,float16,fp8,31,0.07817066709200542
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,1,float16,float16,63,0.08499200145403545
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,1,float16,fp8,63,0.07816533247629802
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,1,float16,fp8,127,0.07783466577529907
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,1,float16,float16,127,0.08635733524958293
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,1,float16,float16,1,0.11571733156840007
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,1,float16,fp8,1,0.0890880028406779
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,1,float16,fp8,3,0.09045333663622539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,1,float16,float16,7,0.11640000343322754
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,1,float16,float16,3,0.11502933502197266
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,1,float16,fp8,7,0.0897706647713979
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,1,float16,float16,15,0.11639466881752014
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,1,float16,float16,31,0.1160533328851064
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,1,float16,fp8,31,0.08944533268610637
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,1,float16,fp8,63,0.08941866954167683
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,1,float16,float16,63,0.11674132943153381
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,1,float16,float16,127,0.1153706709543864
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,1,float16,fp8,127,0.09045333663622539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,1,float16,float16,1,1.4257493019104004
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,1,float16,fp8,1,1.1173546314239502
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,1,float16,float16,3,1.4789973894755046
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,1,float16,fp8,3,1.1603626410166423
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,1,float16,float16,7,1.4984532992045085
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,1,float16,fp8,7,1.2062719662984211
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,1,float16,float16,15,1.776810646057129
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,1,float16,fp8,15,1.5315732955932617
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,1,float16,float16,31,1.8112853368123372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,1,float16,fp8,31,1.5353172620137532
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,1,float16,float16,63,1.8585599263509114
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,1,float16,fp8,63,1.5476053555806477
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,1,float16,float16,1,1.4426453908284504
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,1,float16,fp8,1,1.1238400141398113
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,1,float16,float16,3,1.4895787239074707
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,1,float16,fp8,3,1.1648000081380208
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,1,float16,float16,7,1.5250773429870605
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,1,float16,fp8,7,1.2192533016204834
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,1,float16,float16,15,1.7945599555969238
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,1,float16,fp8,15,1.5380479494730632
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,1,float16,float16,31,1.85480531056722
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,1,float16,fp8,31,1.5435093243916829
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,1,float16,float16,63,1.89081605275472
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,1,float16,float16,1,0.15633066495259604
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,1,float16,fp8,63,1.5660373369852703
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,1,float16,fp8,1,0.145578662554423
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,1,float16,float16,3,0.1556426684061686
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,1,float16,fp8,3,0.1443893313407898
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,1,float16,float16,7,0.15479466319084167
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,1,float16,fp8,7,0.1467680037021637
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,1,float16,float16,15,0.15479466319084167
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,1,float16,fp8,15,0.14523733655611673
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,1,float16,fp8,31,0.14387200276056925
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,1,float16,float16,31,0.15480533242225647
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,1,float16,float16,63,0.15427199999491373
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,1,float16,fp8,63,0.14421866337458292
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,1,float16,float16,1,0.2164106567700704
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,1,float16,float16,3,0.21503466367721558
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,1,float16,fp8,3,0.1662346621354421
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,1,float16,float16,7,0.21538132429122925
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,1,float16,fp8,7,0.16605866948763529
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,1,float16,float16,15,0.2146880030632019
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,1,float16,fp8,15,0.16639999548594156
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,1,float16,float16,31,0.21538132429122925
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,1,float16,fp8,31,0.16639999548594156
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,1,float16,float16,63,0.21538132429122925
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,1,float16,fp8,63,0.16657066345214844
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,1,float16,float16,1,0.01870399961868922
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,1,float16,fp8,1,0.015450666348139444
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,1,float16,float16,3,0.018863999595244724
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,1,float16,fp8,3,0.016730666160583496
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,1,float16,float16,7,0.01887999971707662
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,1,float16,fp8,7,0.01661866654952367
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,1,float16,float16,15,0.02070933332045873
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,1,float16,fp8,15,0.018672000616788864
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,1,float16,float16,31,0.020714666694402695
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,1,float16,fp8,31,0.018858666221300762
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,1,float16,float16,63,0.021055998901526134
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,1,float16,fp8,63,0.018863999595244724
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,1,float16,float16,127,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,1,float16,fp8,127,0.0210506667693456
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,1,float16,float16,255,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,1,float16,fp8,255,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,1,float16,float16,511,0.05358933409055074
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,1,float16,fp8,511,0.04744533201058706
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,1,float16,float16,1023,0.09249599774678548
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,1,float16,fp8,1023,0.08226133386294048
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,1,float16,float16,2047,0.16827734311421713
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,1,float16,fp8,2047,0.1518933375676473
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,1,float16,float16,4095,0.3237546682357788
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,1,float16,fp8,4095,0.2930346727371216
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,1,float16,float16,8191,0.6309546629587809
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,1,float16,fp8,8191,0.5751466751098633
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,1,float16,float16,16383,1.625600020090739
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,1,float16,fp8,16383,1.2441600163777669
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,1,float16,float16,1,0.019007999449968338
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,1,float16,fp8,1,0.016037333756685257
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,1,float16,float16,3,0.01905599981546402
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,1,float16,fp8,3,0.01666133354107539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,1,float16,float16,7,0.019424000134070713
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,1,float16,fp8,7,0.01661866654952367
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,1,float16,float16,15,0.020725333442290623
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,1,float16,fp8,15,0.018863999595244724
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,1,float16,float16,31,0.02072000006834666
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,1,float16,fp8,31,0.018661333868900936
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,1,float16,float16,63,0.021055998901526134
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,1,float16,fp8,63,0.018853332847356796
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,1,float16,float16,127,0.02480533222357432
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,1,float16,fp8,127,0.021045332153638203
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,1,float16,float16,255,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,1,float16,fp8,255,0.031066666046778362
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,1,float16,float16,511,0.05358933409055074
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,1,float16,fp8,511,0.04744533201058706
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,1,float16,float16,1023,0.09250133236249287
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,1,float16,fp8,1023,0.08430932958920796
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,1,float16,float16,2047,0.1687999963760376
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,1,float16,fp8,2047,0.1539413332939148
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,1,float16,float16,4095,0.3271626631418864
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,1,float16,fp8,4095,0.29201600948969525
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,1,float16,float16,8191,0.6775466601053873
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,1,float16,fp8,8191,0.5771946509679159
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,1,float16,float16,16383,1.9140267372131348
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,1,float16,fp8,16383,1.3400746981302898
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,1,float16,float16,1,0.012165332833925882
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,1,float16,fp8,1,0.013482666263977686
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,1,float16,float16,3,0.012629333883523941
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,1,float16,fp8,3,0.0136266661187013
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,1,float16,float16,7,0.01358933374285698
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,1,float16,fp8,7,0.01423466702302297
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,1,float16,float16,15,0.014282666146755219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,1,float16,fp8,15,0.013839999834696451
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,1,float16,float16,31,0.012629333883523941
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,1,float16,fp8,31,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,1,float16,float16,63,0.014053333550691605
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,1,float16,fp8,63,0.013744000345468521
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,1,float16,float16,127,0.012970666090647379
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,1,float16,fp8,127,0.01421333352724711
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,1,float16,float16,255,0.013477332890033722
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,1,float16,fp8,255,0.013477332890033722
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,1,float16,float16,511,0.013301332791646322
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,1,float16,fp8,511,0.012837332983811697
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,1,float16,float16,1023,0.014554666976133982
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,1,float16,fp8,1023,0.014522666732470194
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,1,float16,float16,2047,0.019007999449968338
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,1,float16,fp8,2047,0.01865600049495697
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,1,float16,float16,4095,0.02275199939807256
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,1,float16,fp8,4095,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,1,float16,float16,8191,0.039594667653242745
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,1,float16,fp8,8191,0.027978666126728058
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,1,float16,float16,16383,0.060421332716941833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,1,float16,fp8,16383,0.043354665239652
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,1,float16,float16,1,0.013455999394257864
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,1,float16,fp8,1,0.01393066719174385
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,1,float16,float16,3,0.013679999858140945
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,1,float16,fp8,3,0.012800000607967377
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,1,float16,float16,7,0.013807999591032663
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,1,float16,fp8,7,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,1,float16,float16,15,0.012693333129088083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,1,float16,fp8,15,0.012357333054145178
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,1,float16,float16,31,0.012618667135636011
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,1,float16,fp8,31,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,1,float16,float16,63,0.012634667257467905
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,1,float16,fp8,63,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,1,float16,float16,127,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,1,float16,fp8,127,0.012341332932313284
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,1,float16,float16,255,0.012346666306257248
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,1,float16,fp8,255,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,1,float16,float16,511,0.013194666554530462
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,1,float16,fp8,511,0.012693333129088083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,1,float16,float16,1023,0.014912000546852747
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,1,float16,fp8,1023,0.014741333822409311
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,1,float16,float16,2047,0.022757334013779957
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,1,float16,fp8,2047,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,1,float16,float16,4095,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,1,float16,fp8,4095,0.02492266645034154
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,1,float16,fp8,8191,0.040607998768488564
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,1,float16,float16,16383,0.10171733299891154
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,1,float16,fp8,16383,0.06075199941794077
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,1,float16,float16,1,0.019013332823912304
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,1,float16,fp8,1,0.016234666109085083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,1,float16,float16,3,0.020373333245515823
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,1,float16,fp8,3,0.016469333320856094
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,1,float16,float16,7,0.01889066646496455
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,1,float16,fp8,7,0.016810666769742966
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,1,float16,float16,15,0.020714666694402695
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,1,float16,fp8,15,0.018874666343132656
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,1,float16,float16,31,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,1,float16,fp8,31,0.018672000616788864
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,1,float16,float16,63,0.0210506667693456
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,1,float16,fp8,63,0.018853332847356796
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,1,float16,float16,127,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,1,float16,fp8,127,0.021066665649414062
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,1,float16,float16,255,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,1,float16,fp8,255,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,1,float16,float16,511,0.05358933409055074
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,1,float16,fp8,511,0.04744533201058706
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,1,float16,float16,1023,0.09249599774678548
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,1,float16,fp8,1023,0.08430932958920796
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,1,float16,float16,2047,0.1704960068066915
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,1,float16,fp8,2047,0.15377066532770792
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,1,float16,float16,4095,0.32409600416819256
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,1,float16,fp8,4095,0.2923520008722941
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,1,float16,float16,8191,0.6966666380564371
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,1,float16,fp8,8191,0.575488011042277
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,1,float16,float16,16383,1.911296049753825
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,1,float16,fp8,16383,1.328810691833496
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,1,float16,float16,1,0.013482666263977686
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,1,float16,fp8,1,0.01331199953953425
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,1,float16,float16,3,0.012693333129088083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,1,float16,fp8,3,0.01331199953953425
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,1,float16,float16,7,0.013306666165590286
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,1,float16,fp8,7,0.012362666428089142
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,1,float16,float16,15,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,1,float16,fp8,15,0.013866666704416275
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,1,float16,float16,31,0.012960000584522883
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,1,float16,fp8,31,0.013845333208640417
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,1,float16,float16,63,0.01313599944114685
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,1,float16,fp8,63,0.013754667093356451
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,1,float16,float16,127,0.012800000607967377
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,1,float16,fp8,127,0.013130666067202887
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,1,float16,float16,255,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,1,float16,fp8,255,0.012693333129088083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,1,float16,float16,511,0.014576000471909841
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,1,float16,fp8,511,0.013381333400805792
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,1,float16,fp8,15,0.08840533097585042
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,1,float16,float16,1023,0.014752000570297241
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,1,float16,fp8,1023,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,1,float16,float16,2047,0.020373333245515823
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,1,float16,fp8,2047,0.018698666244745255
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,1,float16,float16,4095,0.022874665757020313
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,1,float16,fp8,4095,0.02252800017595291
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,1,float16,float16,8191,0.03958933303753535
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,1,float16,fp8,8191,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,1,float16,float16,16383,0.06144000093142191
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,1,float16,fp8,16383,0.04266666869322459
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,1,float16,float16,1,0.012357333054145178
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,1,float16,fp8,1,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,1,float16,float16,3,0.012965332716703415
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,1,float16,fp8,3,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,1,float16,float16,7,0.012533333152532578
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,1,float16,fp8,7,0.012560000022252401
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,1,float16,float16,15,0.012693333129088083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,1,float16,fp8,15,0.011205332974592844
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,1,float16,float16,31,0.012351999680201212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,1,float16,fp8,31,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,1,float16,float16,63,0.013807999591032663
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,1,float16,fp8,63,0.01368533323208491
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,1,float16,float16,127,0.013749333719412485
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,1,float16,fp8,127,0.012389333297808966
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,1,float16,float16,255,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,1,float16,fp8,255,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,1,float16,float16,511,0.01302933320403099
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,1,float16,fp8,511,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,1,float16,fp8,1023,0.014607999473810196
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,1,float16,float16,1023,0.014917333920796713
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,1,float16,float16,2047,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,1,float16,fp8,2047,0.0206986665725708
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,1,float16,float16,4095,0.03755199909210205
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,1,float16,fp8,4095,0.025258667767047882
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,1,float16,float16,8191,0.05975466469923655
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,1,float16,fp8,8191,0.039264000952243805
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,1,float16,float16,16383,0.1013759970664978
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,1,float16,fp8,16383,0.06178666651248932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,1,float16,float16,1,0.014746667196353277
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,1,float16,fp8,1,0.014416000495354334
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,1,float16,float16,3,0.014416000495354334
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,1,float16,fp8,3,0.013034666577974955
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,1,float16,float16,7,0.014677333335081736
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,1,float16,fp8,7,0.014416000495354334
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,1,float16,float16,15,0.014479999740918478
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,1,float16,fp8,15,0.012863999853531519
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,1,float16,float16,31,0.014602666099866232
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,1,float16,fp8,31,0.012714666624863943
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,1,float16,float16,63,0.014602666099866232
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,1,float16,fp8,63,0.012869333227475485
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,1,float16,float16,127,0.0129120002190272
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,1,float16,fp8,127,0.012869333227475485
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,1,float16,float16,255,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,1,float16,fp8,255,0.01302933320403099
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,1,float16,float16,511,0.016656000167131424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,1,float16,fp8,511,0.014757333944241205
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,1,float16,float16,1023,0.019066666563351948
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,1,float16,fp8,1023,0.018661333868900936
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,1,float16,fp8,1,0.16587733229001364
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,1,float16,float16,2047,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,1,float16,fp8,2047,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,1,float16,float16,4095,0.06109866499900818
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,1,float16,fp8,4095,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,1,float16,float16,8191,0.10376532872517903
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,1,float16,fp8,8191,0.06177600224812826
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,1,float16,float16,16383,0.1877280076344808
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,1,float16,fp8,16383,0.106495996316274
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,1,float16,float16,1,0.010682666053374609
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,1,float16,fp8,1,0.010597333312034607
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,1,float16,float16,3,0.010922666639089584
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,1,float16,fp8,3,0.010496000448862711
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,1,float16,float16,7,0.009717333440979322
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,1,float16,fp8,7,0.011530666301647821
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,1,float16,float16,15,0.011546666423479715
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,1,float16,fp8,15,0.010922666639089584
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,1,float16,float16,31,0.012383999923865
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,1,float16,fp8,31,0.01145600030819575
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,1,float16,float16,63,0.011551999797423681
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,1,float16,fp8,63,0.012256000190973282
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,1,float16,float16,127,0.011823999385039011
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,1,float16,fp8,127,0.010751999914646149
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,1,float16,float16,255,0.01440000037352244
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,1,float16,fp8,255,0.01444799949725469
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,1,float16,float16,511,0.014581333845853806
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,1,float16,fp8,511,0.014570667097965876
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,1,float16,float16,1023,0.022805333137512207
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,1,float16,float16,2047,0.02900800108909607
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,1,float16,fp8,2047,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,1,float16,float16,4095,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,1,float16,fp8,4095,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,1,float16,float16,8191,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,1,float16,fp8,8191,0.05563733478387197
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,1,float16,float16,16383,0.10069333513577779
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,1,float16,fp8,16383,0.09044266740481059
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,1,float16,float16,32767,0.18278400103251138
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,1,float16,fp8,32767,0.16435199975967407
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,1,float16,float16,65535,0.34457600116729736
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,1,float16,float16,1,0.011317333827416102
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,1,float16,fp8,65535,0.3097599943478902
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,1,float16,fp8,1,0.012693333129088083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,1,float16,float16,3,0.010682666053374609
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,1,float16,fp8,3,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,1,float16,float16,7,0.010693332801262537
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,1,float16,fp8,7,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,1,float16,float16,15,0.011029332876205444
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,1,float16,fp8,15,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,1,float16,fp8,31,0.01251199965675672
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,1,float16,float16,63,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,1,float16,fp8,63,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,1,float16,float16,127,0.011168000598748526
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,1,float16,fp8,127,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,1,float16,float16,255,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,1,float16,fp8,255,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,1,float16,float16,511,0.012351999680201212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,1,float16,fp8,511,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,1,float16,float16,1023,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,1,float16,fp8,1023,0.012698666503032049
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,1,float16,float16,2047,0.012858666479587555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,1,float16,fp8,2047,0.012863999853531519
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,1,float16,float16,4095,0.0144213338692983
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,1,float16,fp8,4095,0.014287999520699183
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,1,float16,float16,8191,0.02070933332045873
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,1,float16,fp8,8191,0.020714666694402695
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,1,float16,float16,16383,0.02629333237806956
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,1,float16,fp8,16383,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,1,float16,float16,32767,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,1,float16,fp8,32767,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,1,float16,float16,65535,0.031397332747777305
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,1,float16,fp8,65535,0.032085334261258446
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,1,float16,float16,1,0.01091733326514562
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,1,float16,fp8,1,0.011663999408483505
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,1,float16,float16,3,0.011941333611806234
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,1,float16,fp8,3,0.012346666306257248
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,1,float16,float16,7,0.011312000453472137
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,1,float16,fp8,7,0.011823999385039011
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,1,float16,float16,15,0.01184533288081487
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,1,float16,fp8,15,0.01251199965675672
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,1,float16,float16,31,0.010821333775917688
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,1,float16,fp8,31,0.012506666282812754
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,1,float16,float16,63,0.010992000500361124
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,1,float16,fp8,63,0.01166933278242747
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,1,float16,float16,127,0.01108266661564509
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,1,float16,fp8,127,0.012005332857370377
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,1,float16,float16,255,0.012170666207869848
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,1,float16,fp8,255,0.012608000387748083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,1,float16,float16,511,0.012149333953857422
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,1,float16,fp8,511,0.01302933320403099
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,1,float16,float16,1023,0.01249066616098086
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,1,float16,fp8,1023,0.01268799975514412
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,1,float16,float16,2047,0.012351999680201212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,1,float16,fp8,2047,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,1,float16,float16,4095,0.012863999853531519
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,1,float16,fp8,4095,0.012858666479587555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,1,float16,float16,8191,0.0186666672428449
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,1,float16,fp8,8191,0.018677332748969395
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,1,float16,float16,16383,0.022757334013779957
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,1,float16,fp8,16383,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,1,float16,float16,32767,0.024911999702453613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,1,float16,fp8,32767,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,1,float16,float16,65535,0.027647999425729115
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,1,float16,float16,8191,0.06007466713587443
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,1,float16,fp8,65535,0.02492800106604894
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,1,float16,float16,1,0.012138667205969492
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,1,float16,fp8,1,0.01240533341964086
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,1,float16,float16,3,0.01180800050497055
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,1,float16,fp8,3,0.012421333541472753
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,1,float16,float16,7,0.012005332857370377
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,1,float16,fp8,7,0.01232533281048139
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,1,float16,float16,15,0.011909333368142446
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,1,float16,fp8,15,0.012186666329701742
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,1,float16,float16,31,0.011605333536863327
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,1,float16,fp8,31,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,1,float16,float16,63,0.011605333536863327
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,1,float16,fp8,63,0.013999999811251959
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,1,float16,float16,127,0.011871999750534693
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,1,float16,fp8,127,0.012709333250919977
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,1,float16,float16,255,0.012682666381200155
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,1,float16,fp8,255,0.01392000044385592
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,1,float16,float16,511,0.012357333054145178
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,1,float16,fp8,511,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,1,float16,float16,1023,0.012682666381200155
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,1,float16,fp8,1023,0.012629333883523941
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,1,float16,float16,2047,0.013274667163689932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,1,float16,fp8,2047,0.014671999961137772
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,1,float16,float16,4095,0.016469333320856094
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,1,float16,fp8,4095,0.016613333175579708
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,1,float16,float16,8191,0.019013332823912304
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,1,float16,fp8,8191,0.018895999838908512
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,1,float16,float16,16383,0.02082666630546252
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,1,float16,fp8,16383,0.020714666694402695
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,1,float16,float16,32767,0.02492266645034154
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,1,float16,fp8,32767,0.02276266614596049
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,1,float16,float16,65535,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,1,float16,fp8,65535,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,1,float16,float16,1,0.011749333391586939
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,1,float16,fp8,1,0.011674666156371435
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,1,float16,float16,3,0.010319999729593595
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,1,float16,fp8,3,0.011626667032639185
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,1,float16,float16,7,0.010538666198650995
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,1,float16,fp8,7,0.011695999652147293
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,1,float16,float16,15,0.011482667177915573
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,1,float16,fp8,15,0.011066666493813196
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,1,float16,float16,31,0.011477333803971609
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,1,float16,fp8,31,0.011642667154471079
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,1,float16,float16,63,0.011391999820868174
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,1,float16,fp8,63,0.012645332763592402
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,1,float16,float16,127,0.011610666910807291
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,1,float16,fp8,127,0.01191466674208641
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,1,float16,float16,255,0.014570667097965876
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,1,float16,fp8,255,0.014736000448465347
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,1,float16,float16,511,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,1,float16,fp8,511,0.022757334013779957
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,1,float16,float16,1023,0.02696000039577484
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,1,float16,fp8,1023,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,1,float16,float16,2047,0.03721066564321518
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,1,float16,fp8,2047,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,1,float16,float16,4095,0.05358933409055074
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,1,float16,fp8,4095,0.04949333270390829
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,1,float16,float16,8191,0.08840533097585042
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,1,float16,fp8,8191,0.08021866778532664
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,1,float16,float16,16383,0.15581867098808289
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,1,float16,fp8,16383,0.14148267110188803
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,1,float16,float16,32767,0.29133333762486774
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,1,float16,fp8,32767,0.26251200834910077
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,1,float16,float16,65535,0.5632053216298422
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,1,float16,fp8,65535,0.504319985707601
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,1,float16,float16,1,0.011973333855470022
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,1,float16,fp8,1,0.011946666985750198
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,1,float16,float16,3,0.012373333175977072
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,1,float16,fp8,3,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,1,float16,float16,7,0.011429333438475927
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,1,float16,fp8,7,0.013584000368913015
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,1,float16,float16,15,0.011952000359694162
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,1,float16,fp8,15,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,1,float16,float16,31,0.01191466674208641
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,1,float16,fp8,31,0.012346666306257248
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,1,float16,float16,63,0.011711999773979187
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,1,float16,fp8,63,0.012144000579913458
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,1,float16,float16,127,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,1,float16,fp8,127,0.012181332955757776
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,1,float16,float16,255,0.012693333129088083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,1,float16,fp8,255,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,1,float16,float16,511,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,1,float16,fp8,511,0.014335999886194864
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,1,float16,float16,1023,0.01268799975514412
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,1,float16,fp8,1023,0.012693333129088083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,1,float16,float16,2047,0.014677333335081736
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,1,float16,fp8,2047,0.014245333770910898
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,1,float16,float16,4095,0.014565333724021912
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,1,float16,fp8,4095,0.014906667172908783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,1,float16,float16,8191,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,1,float16,fp8,8191,0.020725333442290623
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,1,float16,float16,16383,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,1,float16,fp8,16383,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,1,float16,float16,32767,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,1,float16,fp8,32767,0.0266239990790685
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,1,float16,float16,65535,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,1,float16,fp8,65535,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,1,float16,float16,1,0.01116266722480456
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,1,float16,fp8,1,0.012986666212479273
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,1,float16,float16,3,0.0120319997270902
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,1,float16,fp8,3,0.01210133358836174
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,1,float16,float16,7,0.011264000087976456
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,1,float16,fp8,7,0.012437333663304647
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,1,float16,float16,15,0.01211200033624967
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,1,float16,fp8,15,0.01219733307758967
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,1,float16,float16,31,0.01524266724785169
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,1,float16,fp8,31,0.012319999436537424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,1,float16,float16,63,0.0122079998254776
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,1,float16,fp8,63,0.012624000509579977
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,1,float16,float16,127,0.012085333466529846
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,1,float16,fp8,127,0.013493333011865616
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,1,float16,float16,255,0.012304000556468964
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,1,float16,fp8,255,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,1,float16,float16,511,0.014208000153303146
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,1,float16,fp8,511,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,1,float16,float16,1023,0.013829333086808523
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,1,float16,fp8,1023,0.012858666479587555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,1,float16,float16,2047,0.014389333625634512
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,1,float16,fp8,2047,0.012986666212479273
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,1,float16,float16,4095,0.016965333372354507
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,1,float16,fp8,4095,0.01695466662446658
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,1,float16,float16,8191,0.019007999449968338
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,1,float16,fp8,8191,0.020714666694402695
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,1,float16,float16,16383,0.020714666694402695
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,1,float16,fp8,16383,0.02072000006834666
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,1,float16,float16,32767,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,1,float16,fp8,32767,0.02276266614596049
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,1,float16,float16,65535,0.03889599939187368
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,1,float16,fp8,65535,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,1,float16,float16,1,0.012165332833925882
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,1,float16,fp8,1,0.013141332815090815
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,1,float16,float16,3,0.012272000312805176
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,1,float16,fp8,3,0.012458667159080505
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,1,float16,float16,7,0.011754666765530905
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,1,float16,fp8,7,0.01198400060335795
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,1,float16,float16,15,0.011637333780527115
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,1,float16,fp8,15,0.013760000467300415
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,1,float16,float16,31,0.011829332758982977
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,1,float16,fp8,31,0.012096000214417776
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,1,float16,float16,63,0.01211200033624967
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,1,float16,fp8,63,0.012416000167528788
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,1,float16,float16,127,0.011434666812419891
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,1,float16,fp8,127,0.012245333443085352
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,1,float16,float16,255,0.010970667004585266
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,1,float16,fp8,255,0.011839999506870905
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,1,float16,float16,511,0.012346666306257248
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,1,float16,fp8,511,0.013749333719412485
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,1,float16,float16,1023,0.012341332932313284
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,1,float16,fp8,1023,0.01268799975514412
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,1,float16,float16,2047,0.016789333273967106
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,1,float16,float16,4095,0.018672000616788864
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,1,float16,fp8,4095,0.018672000616788864
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,1,float16,float16,8191,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,1,float16,fp8,8191,0.0186666672428449
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,1,float16,float16,16383,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,1,float16,fp8,16383,0.020725333442290623
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,1,float16,fp8,1023,0.021514666577180225
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,1,float16,float16,32767,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,1,float16,fp8,32767,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,1,float16,float16,65535,0.05836800237496694
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,1,float16,float16,1,0.029018667836983997
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,1,float16,fp8,65535,0.03788800040880839
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,1,float16,fp8,1,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,1,float16,float16,3,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,1,float16,fp8,3,0.02492266645034154
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,1,float16,float16,7,0.030378667016824085
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,1,float16,fp8,7,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,1,float16,float16,15,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,1,float16,fp8,15,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,1,float16,float16,31,0.03481066723664602
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,1,float16,fp8,31,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,1,float16,float16,63,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,1,float16,fp8,63,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,1,float16,float16,127,0.04095999896526337
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,1,float16,fp8,127,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,1,float16,float16,255,0.05769599974155426
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,1,float16,fp8,255,0.05358933409055074
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,1,float16,float16,511,0.09454933802286784
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,1,float16,fp8,511,0.08635733524958293
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,1,float16,float16,1023,0.16810667514801025
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,1,float16,float16,31,0.010682666053374609
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,1,float16,fp8,1023,0.15411733587582907
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,1,float16,float16,2047,0.3135146697362264
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,1,float16,fp8,2047,0.28962133328119916
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,1,float16,float16,4095,0.6394879817962646
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,1,float16,fp8,4095,0.5614933172861735
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,1,float16,float16,8191,1.3663573265075684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,1,float16,float16,1,0.012560000022252401
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,1,float16,fp8,8191,1.1125706831614177
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,1,float16,fp8,1,0.013904000322024027
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,1,float16,float16,3,0.012389333297808966
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,1,float16,fp8,3,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,1,float16,float16,7,0.012613333761692047
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,1,float16,fp8,7,0.012970666090647379
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,1,float16,float16,15,0.012655999511480331
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,1,float16,fp8,15,0.013653332988421122
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,1,float16,float16,31,0.012970666090647379
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,1,float16,fp8,31,0.013962666193644205
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,1,float16,float16,63,0.014671999961137772
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,1,float16,fp8,63,0.012448000411192576
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,1,float16,float16,127,0.012698666503032049
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,1,float16,fp8,127,0.012794667234023413
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,1,float16,float16,255,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,1,float16,fp8,255,0.012863999853531519
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,1,float16,float16,511,0.014192000031471252
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,1,float16,fp8,511,0.013077333569526672
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,1,float16,float16,1023,0.016629333297411602
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,1,float16,fp8,1023,0.014949332922697067
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,1,float16,float16,2047,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,1,float16,fp8,2047,0.021040000021457672
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,1,float16,float16,4095,0.038912000755469
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,1,float16,fp8,4095,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,1,float16,float16,8191,0.06177600224812826
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,1,float16,fp8,8191,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,1,float16,float16,1,0.014442666123310724
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,1,float16,fp8,1,0.014432000617186228
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,1,float16,float16,3,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,1,float16,fp8,3,0.0129120002190272
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,1,float16,float16,7,0.014565333724021912
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,1,float16,fp8,7,0.014576000471909841
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,1,float16,float16,15,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,1,float16,fp8,15,0.013034666577974955
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,1,float16,float16,31,0.01461333284775416
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,1,float16,fp8,31,0.012901333471139273
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,1,float16,float16,63,0.014570667097965876
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,1,float16,fp8,63,0.013994666437307993
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,1,float16,float16,127,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,1,float16,fp8,127,0.014570667097965876
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,1,float16,float16,255,0.014325333138306936
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,1,float16,fp8,255,0.012736000120639801
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,1,float16,float16,511,0.016666666915019352
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,1,float16,fp8,511,0.014778666198253632
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,1,float16,float16,1023,0.020768000433842342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,1,float16,fp8,1023,0.01865600049495697
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,1,float16,float16,2047,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,1,float16,fp8,2047,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,1,float16,float16,4095,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,1,float16,fp8,4095,0.041306667029857635
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,1,float16,float16,8191,0.10342400272687276
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,1,float16,fp8,8191,0.0631466656923294
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,1,float16,float16,1,0.016666666915019352
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,1,float16,fp8,1,0.015130666395028433
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,1,float16,float16,3,0.016607999801635742
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,1,float16,fp8,3,0.014783999572197596
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,1,float16,float16,7,0.016656000167131424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,1,float16,fp8,7,0.01672533278663953
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,1,float16,float16,15,0.01666133354107539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,1,float16,fp8,15,0.014757333944241205
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,1,float16,float16,31,0.016965333372354507
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,1,float16,fp8,31,0.014778666198253632
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,1,float16,float16,63,0.016666666915019352
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,1,float16,fp8,63,0.016469333320856094
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,1,float16,float16,127,0.016666666915019352
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,1,float16,fp8,127,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,1,float16,float16,255,0.016672000288963318
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,1,float16,fp8,255,0.014954666296641031
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,1,float16,float16,511,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,1,float16,fp8,511,0.018863999595244724
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,1,float16,float16,1023,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,1,float16,fp8,1023,0.022874665757020313
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,1,float16,float16,2047,0.06177600224812826
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,1,float16,fp8,2047,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,1,float16,float16,4095,0.10478933652242024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,1,float16,fp8,4095,0.06382933259010315
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,1,float16,float16,1,0.010133333504199982
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,1,float16,float16,8191,0.18961066007614136
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,1,float16,fp8,8191,0.10478400190671285
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,1,float16,fp8,1,0.011146667102972666
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,1,float16,float16,3,0.01129066695769628
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,1,float16,fp8,3,0.011503999431928
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,1,float16,float16,7,0.011600000162919363
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,1,float16,fp8,7,0.010304000228643417
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,1,float16,float16,15,0.011141333729028702
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,1,float16,fp8,15,0.011231999844312668
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,1,float16,float16,31,0.011658667276302973
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,1,float16,fp8,31,0.011770666887362799
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,1,float16,float16,63,0.012026666353146235
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,1,float16,fp8,63,0.01232533281048139
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,1,float16,float16,127,0.011333333949247995
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,1,float16,fp8,127,0.012159999459981918
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,1,float16,float16,255,0.022800001005331676
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,1,float16,fp8,255,0.022800001005331676
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,1,float16,float16,511,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,1,float16,fp8,511,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,1,float16,float16,1023,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,1,float16,fp8,1023,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,1,float16,float16,2047,0.04540266593297323
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,1,float16,fp8,2047,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,1,float16,float16,4095,0.07202133536338806
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,1,float16,fp8,4095,0.06587733328342438
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,1,float16,float16,8191,0.12321600317955017
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,1,float16,fp8,8191,0.1109279990196228
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,1,float16,float16,16383,0.22562134265899658
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,1,float16,fp8,16383,0.20292266209920248
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,1,float16,float16,32767,0.43025068442026776
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,1,float16,fp8,32767,0.3858773310979207
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,1,float16,float16,65535,0.9398612976074219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,1,float16,float16,1,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,1,float16,fp8,1,0.011605333536863327
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,1,float16,fp8,65535,0.7488853136698405
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,1,float16,float16,3,0.012341332932313284
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,1,float16,fp8,3,0.012175999581813812
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,1,float16,float16,7,0.011946666985750198
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,1,float16,fp8,7,0.013445333888133367
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,1,float16,float16,15,0.011952000359694162
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,1,float16,fp8,15,0.012693333129088083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,1,float16,float16,31,0.012304000556468964
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,1,float16,fp8,31,0.011434666812419891
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,1,float16,float16,63,0.011264000087976456
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,1,float16,fp8,63,0.012165332833925882
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,1,float16,float16,127,0.012970666090647379
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,1,float16,fp8,127,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,1,float16,float16,255,0.012261333564917246
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,1,float16,float16,511,0.012357333054145178
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,1,float16,fp8,255,0.012383999923865
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,1,float16,fp8,511,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,1,float16,fp8,1023,0.012400000045696894
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,1,float16,float16,1023,0.01268799975514412
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,1,float16,float16,2047,0.01461333284775416
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,1,float16,fp8,2047,0.014511999984582266
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,1,float16,float16,4095,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,1,float16,fp8,4095,0.01893866683046023
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,1,float16,float16,8191,0.021162666380405426
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,1,float16,fp8,8191,0.022426667312781017
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,1,float16,fp8,16383,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,1,float16,float16,32767,0.02696000039577484
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,1,float16,fp8,32767,0.02492266645034154
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,1,float16,float16,65535,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,1,float16,float16,1,0.010816000401973724
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,1,float16,fp8,65535,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,1,float16,fp8,1,0.012159999459981918
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,1,float16,float16,3,0.010858666151762009
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,1,float16,fp8,3,0.01126933346192042
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,1,float16,float16,7,0.010693332801262537
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,1,float16,float16,15,0.010922666639089584
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,1,float16,fp8,7,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,1,float16,fp8,15,0.010858666151762009
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,1,float16,float16,31,0.010650667051474253
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,1,float16,fp8,31,0.011605333536863327
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,1,float16,float16,63,0.010816000401973724
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,1,float16,fp8,63,0.010858666151762009
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,1,float16,float16,127,0.012181332955757776
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,1,float16,fp8,127,0.010858666151762009
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,1,float16,float16,255,0.010816000401973724
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,1,float16,fp8,255,0.011887999872366587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,1,float16,float16,511,0.012346666306257248
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,1,float16,fp8,511,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,1,float16,float16,1023,0.012533333152532578
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,1,float16,fp8,1023,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,1,float16,float16,2047,0.016634666671355564
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,1,float16,fp8,2047,0.016895999511082966
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,1,float16,float16,4095,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,1,float16,fp8,4095,0.01870399961868922
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,1,float16,float16,8191,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,1,float16,fp8,8191,0.02081599955757459
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,1,float16,float16,16383,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,1,float16,fp8,16383,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,1,float16,float16,32767,0.04027199993530909
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,1,float16,fp8,32767,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,1,float16,float16,65535,0.060415998101234436
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,1,float16,float16,1,0.011663999408483505
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,1,float16,fp8,65535,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,1,float16,fp8,1,0.01116266722480456
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,1,float16,float16,3,0.010858666151762009
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,1,float16,fp8,3,0.010992000500361124
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,1,float16,float16,7,0.011882666498422623
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,1,float16,fp8,7,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,1,float16,float16,15,0.01239466667175293
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,1,float16,fp8,15,0.01099733387430509
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,1,float16,float16,31,0.01108266661564509
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,1,float16,fp8,31,0.01232533281048139
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,1,float16,float16,63,0.012223999947309494
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,1,float16,fp8,63,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,1,float16,float16,127,0.011525332927703857
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,1,float16,fp8,127,0.012533333152532578
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,1,float16,float16,255,0.012015999605258306
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,1,float16,fp8,255,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,1,float16,float16,511,0.012448000411192576
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,1,float16,fp8,511,0.01268799975514412
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,1,float16,float16,1023,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,1,float16,fp8,1023,0.012863999853531519
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,1,float16,float16,2047,0.014671999961137772
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,1,float16,fp8,2047,0.014762666076421738
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,1,float16,float16,4095,0.018709332992633183
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,1,float16,fp8,4095,0.016623999923467636
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,1,float16,float16,8191,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,1,float16,fp8,8191,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,1,float16,float16,16383,0.03788800040880839
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,1,float16,fp8,16383,0.024911999702453613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,1,float16,float16,32767,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,1,float16,fp8,32767,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,1,float16,float16,65535,0.10068800052007039
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,1,float16,fp8,65535,0.059392000238100685
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,1,float16,float16,1,0.04949333270390829
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,1,float16,fp8,1,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,1,float16,float16,3,0.05154666801293691
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,1,float16,fp8,3,0.041306667029857635
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,1,float16,float16,7,0.051541333397229515
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,1,float16,fp8,7,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,1,float16,float16,15,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,1,float16,fp8,15,0.05358933409055074
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,1,float16,float16,31,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,1,float16,fp8,31,0.05358933409055074
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,1,float16,float16,63,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,1,float16,fp8,63,0.05358933409055074
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,1,float16,float16,127,0.07406933108965556
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,1,float16,fp8,127,0.06382933259010315
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,1,float16,float16,255,0.10889066259066264
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,1,float16,fp8,255,0.09659733374913533
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,1,float16,float16,511,0.17834667364756265
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,1,float16,fp8,511,0.1599146624406179
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,1,float16,float16,1023,0.31623999277750653
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,1,float16,fp8,1023,0.28893333673477173
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,1,float16,float16,2047,0.6309546629587809
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,1,float16,fp8,2047,0.5492053429285685
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,1,float16,fp8,4095,1.0719679991404216
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,1,float16,float16,4095,1.2794826825459797
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,1,float16,float16,1,0.014912000546852747
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,1,float16,fp8,1,0.014607999473810196
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,1,float16,float16,3,0.01666133354107539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,1,float16,fp8,3,0.01461333284775416
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,1,float16,float16,7,0.01578666642308235
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,1,float16,fp8,7,0.014607999473810196
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,1,float16,float16,15,0.014906667172908783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,1,float16,fp8,15,0.014607999473810196
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,1,float16,float16,31,0.015989333391189575
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,1,float16,fp8,31,0.0145066666106383
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,1,float16,float16,63,0.015018666783968607
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,1,float16,fp8,63,0.014602666099866232
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,1,float16,float16,127,0.015589332828919092
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,1,float16,fp8,127,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,1,float16,float16,255,0.015664000064134598
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,1,float16,fp8,255,0.014511999984582266
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,1,float16,float16,511,0.0186666672428449
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,1,float16,fp8,511,0.01672533278663953
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,1,float16,fp8,1023,0.018698666244745255
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,1,float16,float16,2047,0.04027199993530909
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,1,float16,fp8,2047,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,1,float16,float16,4095,0.0631466656923294
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,1,float16,fp8,4095,0.04539200166861216
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,1,float16,float16,1,0.01666133354107539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,1,float16,fp8,1,0.016672000288963318
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,1,float16,float16,3,0.016666666915019352
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,1,float16,fp8,3,0.016672000288963318
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,1,float16,float16,7,0.016997333616018295
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,1,float16,fp8,7,0.016666666915019352
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,1,float16,float16,15,0.016906666258970898
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,1,float16,fp8,2047,0.016688000410795212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,1,float16,fp8,15,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,1,float16,float16,31,0.01729600007335345
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,1,float16,fp8,31,0.014741333822409311
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,1,float16,float16,63,0.016672000288963318
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,1,float16,fp8,63,0.016672000288963318
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,1,float16,float16,127,0.01729600007335345
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,1,float16,fp8,127,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,1,float16,float16,255,0.01695999999841054
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,1,float16,fp8,255,0.016554666062196095
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,1,float16,float16,511,0.021386665602525074
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,1,float16,fp8,511,0.01878400022784869
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,1,float16,float16,1023,0.03618133316437403
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,1,float16,fp8,1023,0.02492266645034154
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,1,float16,float16,2047,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,1,float16,fp8,2047,0.04199466605981191
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,1,float16,float16,4095,0.10478933652242024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,1,float16,fp8,4095,0.06451733410358429
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,1,float16,float16,1,0.02274133265018463
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,1,float16,fp8,1,0.020138667275508244
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,1,float16,float16,3,0.023103999594847362
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,1,float16,fp8,3,0.018709332992633183
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,1,float16,float16,7,0.023103999594847362
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,1,float16,fp8,7,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,1,float16,float16,15,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,1,float16,fp8,15,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,1,float16,float16,31,0.02309333284695943
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,1,float16,fp8,31,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,1,float16,float16,63,0.02276266614596049
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,1,float16,fp8,63,0.02075200031201045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,1,float16,float16,127,0.022863999009132385
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,1,float16,fp8,127,0.019797333826621372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,1,float16,float16,255,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,1,float16,fp8,255,0.019797333826621372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,1,float16,float16,511,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,1,float16,fp8,511,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,1,float16,float16,1023,0.05462400118509928
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,1,float16,fp8,1023,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,1,float16,float16,2047,0.09762133161226909
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,1,float16,fp8,2047,0.05972800155480703
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,1,float16,float16,4095,0.17663466930389404
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,1,float16,fp8,4095,0.09762133161226909
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,1,float16,float16,1,0.09659733374913533
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,1,float16,fp8,1,0.07611200213432312
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,1,float16,float16,3,0.09659733374913533
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,1,float16,fp8,3,0.07816533247629802
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,1,float16,float16,7,0.09864532947540283
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,1,float16,fp8,7,0.08226133386294048
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,1,float16,float16,15,0.11708266536394756
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,1,float16,fp8,15,0.10274133086204529
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,1,float16,float16,31,0.11912533640861511
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,1,float16,fp8,31,0.10274133086204529
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,1,float16,float16,63,0.11912533640861511
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,1,float16,fp8,63,0.10273599624633789
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,1,float16,float16,127,0.14148267110188803
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,1,float16,fp8,127,0.12117333213488261
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,1,float16,float16,255,0.20940800507863364
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,1,float16,fp8,255,0.18449066082636514
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,1,float16,float16,511,0.34867199261983234
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,1,float16,fp8,511,0.3104479908943176
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,1,float16,float16,1023,0.6531466643015543
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,1,float16,fp8,1023,0.5672959884007772
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,1,float16,float16,1,0.02070933332045873
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,1,float16,float16,3,0.020703999946514767
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,1,float16,fp8,1,0.01870399961868922
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,1,float16,float16,7,0.02075200031201045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,1,float16,fp8,7,0.018709332992633183
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,1,float16,float16,15,0.019098666807015736
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,1,float16,fp8,15,0.018789333601792652
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,1,float16,float16,31,0.019402666638294857
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,1,float16,fp8,31,0.018709332992633183
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,1,float16,float16,63,0.02081599955757459
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,1,float16,fp8,63,0.01870399961868922
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,1,float16,float16,127,0.02075200031201045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,1,float16,fp8,127,0.018725333114465077
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,1,float16,float16,255,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,1,float16,fp8,255,0.018016000588734944
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,1,float16,float16,511,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,1,float16,fp8,511,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,1,float16,float16,1023,0.03857066730658213
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,1,float16,fp8,1023,0.026975999275843304
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,1,float16,float16,1,0.023210667073726654
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,1,float16,fp8,1,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,1,float16,float16,3,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,1,float16,fp8,3,0.020410666863123577
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,1,float16,float16,7,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,1,float16,fp8,7,0.020138667275508244
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,1,float16,float16,15,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,1,float16,fp8,15,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,1,float16,float16,31,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,1,float16,fp8,31,0.02082666630546252
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,1,float16,float16,63,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,1,float16,fp8,63,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,1,float16,float16,127,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,1,float16,fp8,127,0.019109333554903667
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,1,float16,float16,255,0.023893333971500397
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,1,float16,fp8,255,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,1,float16,float16,511,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,1,float16,fp8,511,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,1,float16,float16,1023,0.05571199953556061
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,1,float16,fp8,1023,0.03719999889532725
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,1,float16,float16,16383,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,1,float16,float16,1,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,1,float16,fp8,1,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,1,float16,float16,3,0.035162667433420815
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,1,float16,fp8,3,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,1,float16,float16,7,0.03379199902216593
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,1,float16,fp8,7,0.02900800108909607
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,1,float16,float16,15,0.03378133227427801
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,1,float16,fp8,15,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,1,float16,float16,31,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,1,float16,fp8,31,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,1,float16,float16,63,0.033786666889985405
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,1,float16,fp8,63,0.02867199977238973
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,1,float16,float16,127,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,1,float16,fp8,127,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,1,float16,float16,255,0.03549333413441976
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,1,float16,fp8,255,0.027653334041436512
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,1,float16,float16,511,0.05632533133029938
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,1,float16,fp8,511,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,1,float16,float16,1023,0.09489066402117412
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,1,float16,fp8,1023,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,1,float16,float16,1,0.18414932489395142
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,1,float16,fp8,1,0.14591999848683676
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,1,float16,float16,3,0.18737600247065225
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,1,float16,fp8,3,0.14984533190727234
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,1,float16,float16,7,0.19165867567062378
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,1,float16,fp8,7,0.15786666671435037
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,1,float16,float16,15,0.22766933838526407
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,1,float16,fp8,15,0.19883199532826742
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,1,float16,float16,31,0.2293813427289327
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,1,float16,fp8,31,0.19746132691701254
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,1,float16,float16,63,0.22971733411153158
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,1,float16,fp8,63,0.19882667064666748
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,1,float16,float16,127,0.27460267146428424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,1,float16,fp8,127,0.23415466149648032
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,1,float16,float16,255,0.40960001945495605
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,1,float16,fp8,255,0.3619840145111084
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,1,float16,float16,511,0.7222613493601481
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,1,float16,fp8,511,0.6133706569671631
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,1,float16,float16,1,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,1,float16,fp8,1,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,1,float16,float16,3,0.02900800108909607
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,1,float16,fp8,3,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,1,float16,float16,7,0.02900800108909607
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,1,float16,fp8,7,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,1,float16,float16,15,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,1,float16,fp8,15,0.02697066714366277
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,1,float16,float16,31,0.029018667836983997
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,1,float16,fp8,31,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,1,float16,float16,63,0.02900800108909607
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,1,float16,fp8,63,0.02696000039577484
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,1,float16,float16,127,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,1,float16,fp8,127,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,1,float16,float16,255,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,1,float16,fp8,255,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,1,float16,float16,511,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,1,float16,fp8,511,0.03311999887228012
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,1,float16,fp8,1,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,1,float16,float16,3,0.03549866626660029
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,1,float16,fp8,3,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,1,float16,float16,7,0.03524799893299738
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,1,float16,fp8,7,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,1,float16,float16,15,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,1,float16,fp8,15,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,1,float16,float16,31,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,1,float16,fp8,31,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,1,float16,float16,63,0.03549866626660029
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,1,float16,fp8,63,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,1,float16,float16,127,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,1,float16,fp8,127,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,1,float16,float16,255,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,1,float16,fp8,255,0.0314026673634847
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,1,float16,float16,511,0.05962666869163513
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,1,float16,fp8,511,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,1,float16,float16,1,0.0576853354771932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,1,float16,fp8,1,0.04744533201058706
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,1,float16,float16,3,0.0576853354771932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,1,float16,fp8,3,0.04642133414745331
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,1,float16,float16,7,0.05905066430568695
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,1,float16,float16,1023,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,1,float16,fp8,7,0.045738667249679565
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,1,float16,float16,15,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,1,float16,fp8,15,0.04710400104522705
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,1,float16,float16,31,0.0580320010582606
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,1,float16,fp8,31,0.04538666705290476
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,1,float16,float16,63,0.05972800155480703
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,1,float16,fp8,63,0.046762665112813316
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,1,float16,float16,127,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,1,float16,fp8,127,0.04539733131726583
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,1,float16,float16,255,0.06177600224812826
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,1,float16,fp8,255,0.04778666794300079
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,1,float16,float16,511,0.0986400047938029
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,1,float16,float16,1,0.014773332824309668
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,1,float16,fp8,511,0.0692853331565857
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,1,float16,fp8,1,0.011626667032639185
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,1,float16,float16,3,0.01461333284775416
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,1,float16,fp8,3,0.011429333438475927
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,1,float16,float16,7,0.01461333284775416
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,1,float16,fp8,7,0.011194666226704916
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,1,float16,float16,15,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,1,float16,fp8,15,0.011839999506870905
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,1,float16,float16,31,0.016672000288963318
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,1,float16,fp8,31,0.012703999876976013
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,1,float16,float16,63,0.01672533278663953
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,1,float16,fp8,63,0.012698666503032049
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,1,float16,float16,127,0.016672000288963318
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,1,float16,fp8,127,0.01303999995191892
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,1,float16,float16,255,0.018624000251293182
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,1,float16,fp8,255,0.014688000082969666
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,1,float16,float16,511,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,1,float16,fp8,511,0.019007999449968338
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,1,float16,float16,1023,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,1,float16,fp8,1023,0.02900800108909607
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,1,float16,float16,2047,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,1,float16,fp8,2047,0.049498667319615684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,1,float16,float16,4095,0.1088853379090627
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,1,float16,fp8,4095,0.08840533097585042
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,1,float16,float16,8191,0.2053119937578837
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,1,float16,fp8,8191,0.16622400283813477
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,1,float16,float16,16383,0.3973226547241211
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,1,float16,fp8,16383,0.32204800844192505
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,1,float16,float16,32767,0.8070826530456543
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,1,float16,fp8,32767,0.6775519847869873
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,1,float16,float16,65535,2.2043306032816568
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,1,float16,fp8,65535,1.5458933512369792
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,1,float16,float16,1,0.012346666306257248
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,1,float16,fp8,1,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,1,float16,float16,3,0.013253333667914072
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,1,float16,fp8,3,0.01313599944114685
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,1,float16,float16,7,0.012261333564917246
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,1,float16,fp8,7,0.012346666306257248
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,1,float16,float16,15,0.012655999511480331
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,1,float16,fp8,15,0.011957333733638128
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,1,float16,float16,31,0.013072000195582708
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,1,float16,fp8,31,0.014090667168299357
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,1,float16,float16,63,0.013082666943470636
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,1,float16,fp8,63,0.012015999605258306
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,1,float16,float16,127,0.012240000069141388
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,1,float16,fp8,127,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,1,float16,float16,255,0.012458667159080505
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,1,float16,fp8,255,0.012970666090647379
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,1,float16,float16,511,0.013829333086808523
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,1,float16,fp8,511,0.012874666601419449
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,1,float16,float16,1023,0.01369599997997284
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,1,float16,fp8,1023,0.012698666503032049
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,1,float16,float16,2047,0.01871466636657715
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,1,float16,fp8,2047,0.018661333868900936
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,1,float16,float16,4095,0.018944000204404194
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,1,float16,fp8,4095,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,1,float16,float16,8191,0.022618666291236877
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,1,float16,fp8,8191,0.021061333517233532
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,1,float16,float16,16383,0.024847999215126038
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,1,float16,fp8,16383,0.024911999702453613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,1,float16,float16,32767,0.041984001795450844
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,1,float16,fp8,32767,0.030720000465710957
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,1,float16,fp8,65535,0.04369066655635834
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,1,float16,float16,65535,0.06144000093142191
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,1,float16,float16,1,0.013674666484196981
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,1,float16,fp8,1,0.011205332974592844
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,1,float16,float16,3,0.013178666432698568
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,1,float16,fp8,3,0.011535999675591787
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,1,float16,float16,7,0.012362666428089142
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,1,float16,fp8,7,0.012117333710193634
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,1,float16,float16,15,0.01184533288081487
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,1,float16,fp8,15,0.011039999624093374
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,1,float16,float16,31,0.012133333832025528
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,1,float16,fp8,31,0.012154666086037954
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,1,float16,fp8,3,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,1,float16,float16,63,0.012970666090647379
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,1,float16,fp8,63,0.012047999848922094
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,1,float16,fp8,127,0.012144000579913458
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,1,float16,float16,127,0.012170666207869848
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,1,float16,float16,255,0.011642667154471079
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,1,float16,fp8,255,0.012693333129088083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,1,float16,float16,511,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,1,float16,fp8,511,0.012400000045696894
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,1,float16,float16,1023,0.012869333227475485
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,1,float16,float16,2047,0.016458666572968166
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,1,float16,fp8,1023,0.012863999853531519
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,1,float16,fp8,2047,0.016623999923467636
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,1,float16,float16,4095,0.016677333662907284
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,1,float16,fp8,4095,0.016821333517630894
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,1,float16,float16,8191,0.02276266614596049
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,1,float16,fp8,8191,0.020410666863123577
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,1,float16,float16,16383,0.03857066730658213
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,1,float16,fp8,16383,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,1,float16,float16,32767,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,1,float16,fp8,32767,0.039621333281199135
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,1,float16,float16,65535,0.10069333513577779
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,1,float16,float16,1,0.012853333105643591
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,1,float16,fp8,65535,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,1,float16,fp8,1,0.011605333536863327
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,1,float16,float16,3,0.01251199965675672
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,1,float16,fp8,3,0.012730666746695837
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,1,float16,float16,7,0.01201066623131434
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,1,float16,fp8,7,0.011765333513418833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,1,float16,float16,15,0.012256000190973282
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,1,float16,fp8,15,0.012346666306257248
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,1,float16,float16,31,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,1,float16,fp8,31,0.01301866645614306
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,1,float16,float16,63,0.012437333663304647
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,1,float16,fp8,63,0.012746666868527731
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,1,float16,float16,127,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,1,float16,fp8,127,0.012693333129088083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,1,float16,float16,255,0.012261333564917246
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,1,float16,fp8,255,0.012138667205969492
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,1,float16,float16,511,0.012693333129088083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,1,float16,fp8,511,0.013306666165590286
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,1,float16,float16,1023,0.014607999473810196
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,1,float16,fp8,1023,0.0144213338692983
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,1,float16,float16,2047,0.01884799947341283
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,1,float16,fp8,2047,0.016528000434239704
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,1,float16,float16,4095,0.021055998901526134
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,1,float16,fp8,4095,0.020714666694402695
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,1,float16,float16,8191,0.03822399924198786
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,1,float16,fp8,8191,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,1,float16,float16,16383,0.059392000238100685
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,1,float16,float16,32767,0.10069333513577779
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,1,float16,fp8,32767,0.05972800155480703
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,1,float16,float16,65535,0.18346667289733887
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,1,float16,fp8,65535,0.10035199920336406
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,1,float16,float16,1,0.36061867078145343
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,1,float16,fp8,1,0.28484266996383667
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,1,float16,float16,3,0.36955201625823975
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,1,float16,fp8,3,0.29337600866953534
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,1,float16,float16,7,0.3773386478424072
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,1,float16,fp8,7,0.3094133337338765
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,1,float16,float16,15,0.44868266582489014
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,1,float16,fp8,15,0.39031465848286945
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,1,float16,float16,31,0.4527786572774251
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,1,float16,fp8,31,0.38929065068562824
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,1,float16,float16,63,0.4551733334859212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,1,float16,fp8,63,0.3916800022125244
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,1,float16,float16,127,0.5454506476720175
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,1,float16,fp8,127,0.4613120158513387
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,1,float16,float16,255,0.8625493049621582
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,1,float16,fp8,255,0.7164533138275146
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,1,float16,float16,1,0.04540266593297323
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,1,float16,fp8,1,0.04540266593297323
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,1,float16,float16,3,0.04539733131726583
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,1,float16,fp8,3,0.04539733131726583
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,1,float16,float16,7,0.04744533201058706
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,1,float16,fp8,7,0.04539733131726583
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,1,float16,float16,15,0.04677333434422811
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,1,float16,fp8,15,0.04539733131726583
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,1,float16,float16,31,0.04642133414745331
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,1,float16,fp8,31,0.04540266593297323
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,1,float16,float16,63,0.045738667249679565
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,1,float16,fp8,63,0.04539733131726583
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,1,float16,float16,127,0.047450666626294456
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,1,float16,fp8,127,0.04370133578777313
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,1,float16,float16,255,0.048810665806134544
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,1,float16,fp8,255,0.043007999658584595
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,1,float16,float16,1,0.06382933259010315
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,1,float16,fp8,1,0.051541333397229515
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,1,float16,fp8,3,0.051541333397229515
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,1,float16,float16,7,0.06417066852251689
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,1,float16,fp8,7,0.051541333397229515
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,1,float16,float16,15,0.06382933259010315
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,1,float16,fp8,15,0.051541333397229515
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,1,float16,float16,31,0.06417066852251689
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,1,float16,fp8,31,0.05153599878152212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,1,float16,float16,63,0.06348266700903575
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,1,float16,fp8,63,0.051541333397229515
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,1,float16,float16,127,0.0641653339068095
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,1,float16,fp8,127,0.051541333397229515
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,1,float16,float16,255,0.06587733328342438
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,1,float16,fp8,255,0.05563200016816457
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,1,float16,float16,1,0.1088853379090627
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,1,float16,fp8,1,0.08397866288820903
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,1,float16,float16,3,0.1088853379090627
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,1,float16,fp8,3,0.08396266897519429
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,1,float16,float16,7,0.10820266604423523
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,1,float16,float16,1,0.03719999889532725
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,1,float16,fp8,7,0.08431466420491536
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,1,float16,float16,15,0.10889066259066264
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,1,float16,fp8,15,0.08293333152929942
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,1,float16,float16,31,0.1088800032933553
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,1,float16,fp8,31,0.08226133386294048
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,1,float16,float16,63,0.10853866736094157
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,1,float16,fp8,63,0.08328000207742055
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,1,float16,float16,127,0.1088853379090627
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,1,float16,fp8,127,0.08430932958920796
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,1,float16,float16,255,0.1088853379090627
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,1,float16,fp8,255,0.0846560001373291
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,1,float16,float16,1,0.7133866945902506
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,1,float16,fp8,1,0.5611519813537598
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,1,float16,fp8,3,0.5799200137456259
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,1,float16,float16,3,0.7335253556569418
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,1,float16,fp8,7,0.6079146862030029
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,1,float16,float16,7,0.7502400080362955
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,1,float16,fp8,15,0.7738133271535238
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,1,float16,float16,15,0.8891733487447103
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,1,float16,fp8,31,0.7703946431477865
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,1,float16,float16,31,0.9024853706359863
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,1,float16,fp8,63,0.7744853496551514
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,1,float16,float16,63,0.9210933049519857
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,1,float16,fp8,127,0.9285973707834879
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,1,float16,float16,127,1.1661653518676758
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,1,float16,float16,1,0.08260266482830048
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,1,float16,fp8,1,0.07815999786059062
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,1,float16,float16,3,0.08294400076071422
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,1,float16,fp8,3,0.07816533247629802
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,1,float16,float16,7,0.08396800359090169
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,1,float16,fp8,7,0.07816533247629802
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,1,float16,float16,15,0.08362666765848796
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,1,float16,fp8,15,0.07850666840871175
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,1,float16,fp8,31,0.07815999786059062
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,1,float16,float16,31,0.08328533172607422
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,1,float16,fp8,63,0.07714133461316426
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,1,float16,float16,63,0.08294400076071422
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,1,float16,fp8,127,0.07679999868075053
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,1,float16,float16,1,0.11502933502197266
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,1,float16,fp8,1,0.09148800373077393
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,1,float16,float16,3,0.1153706709543864
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,1,float16,fp8,3,0.09045333663622539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,1,float16,float16,7,0.1153706709543864
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,1,float16,fp8,7,0.09114133318265279
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,1,float16,float16,15,0.11707199613253276
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,1,float16,fp8,15,0.09113066395123799
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,1,float16,float16,31,0.11502933502197266
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,1,float16,fp8,31,0.09045867125193278
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,1,float16,float16,63,0.1153706709543864
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,1,float16,fp8,63,0.09079466263453166
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,1,float16,float16,127,0.1153706709543864
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,1,float16,fp8,127,0.09250133236249287
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,1,float16,float16,1,0.20309333006540933
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,1,float16,fp8,1,0.15411200126012167
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,1,float16,float16,3,0.2036959926287333
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,1,float16,fp8,3,0.15377066532770792
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,1,float16,float16,7,0.20241600275039673
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,1,float16,fp8,7,0.15377066532770792
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,1,float16,float16,15,0.20292266209920248
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,1,float16,fp8,15,0.15307733416557312
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,1,float16,float16,31,0.2032639980316162
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,1,float16,fp8,31,0.1539413332939148
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,1,float16,float16,63,0.20309333006540933
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,1,float16,fp8,63,0.15275733669598898
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,1,float16,float16,127,0.2032639980316162
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,1,float16,fp8,127,0.15241066614786783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,1,float16,float16,1,0.014773332824309668
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,1,float16,fp8,1,0.016613333175579708
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,1,float16,float16,3,0.014858666807413101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,1,float16,fp8,3,0.01661866654952367
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,1,float16,float16,7,0.01462399959564209
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,1,float16,fp8,7,0.016677333662907284
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,1,float16,float16,15,0.014912000546852747
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,1,float16,fp8,15,0.01695466662446658
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,1,float16,float16,31,0.015082667271296183
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,1,float16,fp8,16383,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,1,float16,fp8,31,0.02070933332045873
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,1,float16,float16,63,0.014783999572197596
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,1,float16,fp8,63,0.02070933332045873
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,1,float16,float16,127,0.016656000167131424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,1,float16,fp8,127,0.02072000006834666
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,1,float16,float16,255,0.022800001005331676
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,1,float16,fp8,255,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,1,float16,float16,511,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,1,float16,fp8,511,0.03380800038576126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,1,float16,float16,1023,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,1,float16,fp8,1023,0.05358933409055074
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,1,float16,float16,2047,0.10683733224868774
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,1,float16,fp8,2047,0.09044800202051799
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,1,float16,float16,4095,0.20189332962036133
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,1,float16,fp8,4095,0.16776533921559653
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,1,float16,float16,8191,0.3916800022125244
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,1,float16,fp8,8191,0.3199999928474426
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,1,float16,fp8,16383,0.6263466676076254
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,1,float16,float16,16383,0.8108373483022054
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,1,float16,fp8,32767,1.377962589263916
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,1,float16,float16,32767,3.6041386922200522
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,1,float16,float16,1,0.01232533281048139
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,1,float16,fp8,1,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,1,float16,float16,3,0.012202666451533636
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,1,float16,fp8,3,0.01268799975514412
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,1,float16,float16,7,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,1,float16,fp8,7,0.012351999680201212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,1,float16,float16,15,0.01228800043463707
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,1,float16,fp8,15,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,1,float16,float16,31,0.012693333129088083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,1,float16,fp8,31,0.012351999680201212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,1,float16,float16,63,0.012053333222866058
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,1,float16,fp8,63,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,1,float16,float16,127,0.012282667060693106
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,1,float16,fp8,127,0.012671999633312225
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,1,float16,float16,255,0.013194666554530462
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,1,float16,fp8,255,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,1,float16,float16,511,0.012560000022252401
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,1,float16,fp8,511,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,1,float16,float16,1023,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,1,float16,float16,2047,0.016656000167131424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,1,float16,fp8,2047,0.016549333930015564
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,1,float16,float16,4095,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,1,float16,fp8,4095,0.018709332992633183
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,1,float16,float16,8191,0.022805333137512207
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,1,float16,float16,3,0.06382933259010315
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,1,float16,fp8,8191,0.02218666672706604
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,1,float16,float16,16383,0.039247999588648476
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,1,float16,fp8,16383,0.026885333160559338
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,1,float16,float16,32767,0.060309335589408875
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,1,float16,fp8,32767,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,1,float16,float16,1,0.010992000500361124
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,1,float16,fp8,1,0.012223999947309494
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,1,float16,float16,3,0.012351999680201212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,1,float16,fp8,3,0.012400000045696894
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,1,float16,float16,7,0.011125333607196808
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,1,float16,fp8,7,0.012223999947309494
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,1,float16,float16,15,0.012351999680201212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,1,float16,fp8,15,0.012047999848922094
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,1,float16,float16,31,0.012341332932313284
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,1,float16,fp8,31,0.011722666521867117
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,1,float16,float16,63,0.012778667112191519
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,1,float16,fp8,63,0.01211200033624967
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,1,float16,float16,127,0.012357333054145178
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,1,float16,fp8,127,0.011776000261306763
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,1,float16,float16,255,0.012351999680201212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,1,float16,fp8,255,0.012144000579913458
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,1,float16,float16,511,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,1,float16,fp8,511,0.012560000022252401
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,1,float16,float16,1023,0.012869333227475485
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,1,float16,fp8,1023,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,1,float16,float16,2047,0.018661333868900936
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,1,float16,fp8,2047,0.016901332885026932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,1,float16,float16,4095,0.022181332111358643
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,1,float16,fp8,4095,0.02075200031201045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,1,float16,float16,8191,0.03851733356714249
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,1,float16,fp8,8191,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,1,float16,float16,16383,0.05972800155480703
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,1,float16,fp8,16383,0.039594667653242745
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,1,float16,float16,32767,0.10103467106819153
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,1,float16,fp8,32767,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,1,float16,float16,1,0.01268799975514412
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,1,float16,fp8,1,0.012421333541472753
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,1,float16,float16,3,0.012698666503032049
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,1,float16,fp8,3,0.013663999736309052
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,1,float16,float16,7,0.01249066616098086
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,1,float16,fp8,7,0.011429333438475927
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,1,float16,float16,15,0.012362666428089142
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,1,float16,fp8,15,0.012389333297808966
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,1,float16,float16,31,0.012693333129088083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,1,float16,fp8,31,0.01302933320403099
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,1,float16,float16,63,0.012181332955757776
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,1,float16,fp8,63,0.012970666090647379
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,1,float16,float16,127,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,1,float16,fp8,127,0.012624000509579977
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,1,float16,float16,255,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,1,float16,fp8,255,0.012383999923865
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,1,float16,float16,511,0.013034666577974955
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,1,float16,fp8,511,0.012730666746695837
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,1,float16,float16,1023,0.014965333044528961
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,1,float16,fp8,1023,0.01444799949725469
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,1,float16,float16,2047,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,1,float16,fp8,2047,0.020762667059898376
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,1,float16,float16,4095,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,1,float16,fp8,4095,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,1,float16,float16,8191,0.060080001751581825
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,1,float16,fp8,8191,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,1,float16,float16,16383,0.10274133086204529
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,1,float16,fp8,16383,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,1,float16,float16,32767,0.186191995938619
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,1,float16,fp8,32767,0.10274666547775269
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,1,float16,float16,1,0.012586666891972223
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,1,float16,fp8,1,0.01239466667175293
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,1,float16,float16,3,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,1,float16,fp8,3,0.012389333297808966
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,1,float16,float16,7,0.012719999998807907
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,1,float16,fp8,7,0.012223999947309494
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,1,float16,float16,15,0.012533333152532578
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,1,float16,fp8,15,0.01239466667175293
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,1,float16,float16,31,0.012970666090647379
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,1,float16,fp8,31,0.012389333297808966
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,1,float16,float16,63,0.012357333054145178
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,1,float16,fp8,63,0.01340266689658165
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,1,float16,float16,127,0.01251199965675672
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,1,float16,fp8,127,0.012015999605258306
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,1,float16,float16,255,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,1,float16,fp8,255,0.012613333761692047
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,1,float16,float16,511,0.01303999995191892
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,1,float16,fp8,511,0.012719999998807907
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,1,float16,float16,1023,0.013034666577974955
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,1,float16,fp8,1023,0.012901333471139273
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,1,float16,float16,2047,0.016965333372354507
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,1,float16,fp8,2047,0.016666666915019352
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,1,float16,float16,4095,0.019023999571800232
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,1,float16,fp8,4095,0.01869333287080129
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,1,float16,float16,8191,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,1,float16,fp8,8191,0.02252800017595291
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,1,float16,float16,16383,0.04095999896526337
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,1,float16,fp8,16383,0.026954665780067444
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,1,float16,float16,32767,0.062463998794555664
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,1,float16,fp8,32767,0.043338666359583534
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,1,float16,float16,1,0.012698666503032049
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,1,float16,fp8,1,0.013573333621025085
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,1,float16,float16,3,0.012351999680201212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,1,float16,fp8,3,0.012602667013804117
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,1,float16,float16,7,0.012319999436537424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,1,float16,fp8,7,0.012186666329701742
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,1,float16,float16,15,0.012362666428089142
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,1,float16,fp8,15,0.012629333883523941
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,1,float16,float16,31,0.012373333175977072
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,1,float16,fp8,31,0.012159999459981918
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,1,float16,float16,63,0.012485332787036896
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,1,float16,fp8,63,0.012223999947309494
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,1,float16,float16,127,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,1,float16,fp8,127,0.012629333883523941
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,1,float16,float16,255,0.01231466606259346
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,1,float16,fp8,255,0.012869333227475485
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,1,float16,float16,127,0.08601066470146179
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,1,float16,float16,511,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,1,float16,fp8,511,0.012719999998807907
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,1,float16,float16,1023,0.012917333592971167
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,1,float16,fp8,1023,0.014576000471909841
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,1,float16,float16,2047,0.01871466636657715
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,1,float16,fp8,2047,0.018698666244745255
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,1,float16,float16,4095,0.02276266614596049
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,1,float16,fp8,4095,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,1,float16,float16,8191,0.03822399924198786
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,1,float16,fp8,8191,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,1,float16,float16,16383,0.060415998101234436
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,1,float16,fp8,16383,0.03925866633653641
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,1,float16,float16,32767,0.10171733299891154
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,1,float16,fp8,32767,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,1,float16,float16,1,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,1,float16,fp8,1,0.01357866699496905
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,1,float16,float16,3,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,1,float16,fp8,3,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,1,float16,float16,7,0.012357333054145178
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,1,float16,fp8,7,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,1,float16,float16,15,0.01228800043463707
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,1,float16,fp8,15,0.01251199965675672
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,1,float16,float16,31,0.011264000087976456
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,1,float16,fp8,31,0.013914667069911957
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,1,float16,float16,63,0.012858666479587555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,1,float16,fp8,63,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,1,float16,float16,127,0.01239466667175293
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,1,float16,fp8,127,0.012351999680201212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,1,float16,float16,255,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,1,float16,fp8,255,0.01239466667175293
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,1,float16,float16,511,0.013210666676362356
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,1,float16,fp8,511,0.012784000486135483
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,1,float16,float16,1023,0.014773332824309668
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,1,float16,fp8,1023,0.014570667097965876
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,1,float16,float16,2047,0.022863999009132385
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,1,float16,fp8,2047,0.02048533285657565
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,1,float16,float16,4095,0.039594667653242745
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,1,float16,fp8,4095,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,1,float16,float16,8191,0.060415998101234436
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,1,float16,fp8,8191,0.039936001102129616
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,1,float16,fp8,16383,0.061093335350354515
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,1,float16,float16,16383,0.10171733299891154
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,1,float16,float16,1,0.012885333349307379
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,1,float16,float16,32767,0.1848319967587789
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,1,float16,fp8,32767,0.10478933652242024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,1,float16,fp8,1,0.013482666263977686
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,1,float16,float16,3,0.014757333944241205
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,1,float16,fp8,3,0.013077333569526672
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,1,float16,float16,7,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,1,float16,fp8,7,0.012586666891972223
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,1,float16,float16,15,0.012906666845083237
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,1,float16,float16,31,0.013541333377361298
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,1,float16,fp8,15,0.012736000120639801
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,1,float16,fp8,31,0.012901333471139273
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,1,float16,float16,63,0.014736000448465347
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,1,float16,fp8,63,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,1,float16,float16,127,0.014607999473810196
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,1,float16,fp8,127,0.01303999995191892
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,1,float16,float16,255,0.014677333335081736
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,1,float16,fp8,255,0.013248000293970108
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,1,float16,float16,511,0.01661866654952367
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,1,float16,fp8,511,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,1,float16,float16,1023,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,1,float16,fp8,1023,0.018661333868900936
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,1,float16,fp8,2047,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,1,float16,float16,2047,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,1,float16,float16,4095,0.062122667829195656
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,1,float16,fp8,4095,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,1,float16,float16,8191,0.10410666465759277
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,1,float16,fp8,8191,0.062463998794555664
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,1,float16,float16,16383,0.18755733966827393
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,1,float16,fp8,16383,0.10513599713643391
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,1,float16,float16,32767,0.35822399457295734
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,1,float16,float16,1,0.011264000087976456
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,1,float16,fp8,32767,0.1909760038057963
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,1,float16,fp8,1,0.012063999970753988
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,1,float16,float16,3,0.012693333129088083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,1,float16,fp8,3,0.012389333297808966
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,1,float16,float16,7,0.012245333443085352
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,1,float16,fp8,7,0.012346666306257248
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,1,float16,float16,15,0.012351999680201212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,1,float16,fp8,15,0.01190399999419848
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,1,float16,float16,31,0.01109333336353302
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,1,float16,fp8,31,0.012272000312805176
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,1,float16,fp8,63,0.012357333054145178
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,1,float16,float16,63,0.010693332801262537
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,1,float16,float16,127,0.01227733368674914
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,1,float16,fp8,127,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,1,float16,float16,255,0.010682666053374609
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,1,float16,fp8,255,0.01240533341964086
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,1,float16,float16,511,0.01268799975514412
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,1,float16,fp8,511,0.012725333372751871
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,1,float16,float16,1023,0.012448000411192576
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,1,float16,fp8,1023,0.012863999853531519
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,1,float16,float16,2047,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,1,float16,fp8,2047,0.012906666845083237
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,1,float16,float16,4095,0.014576000471909841
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,1,float16,fp8,4095,0.01392000044385592
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,1,float16,float16,8191,0.018853332847356796
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,1,float16,fp8,8191,0.02070933332045873
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,1,float16,float16,16383,0.02492266645034154
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,1,float16,fp8,16383,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,1,float16,float16,32767,0.03721066564321518
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,1,float16,fp8,32767,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,1,float16,float16,65535,0.041984001795450844
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,1,float16,fp8,65535,0.041984001795450844
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,1,float16,float16,131071,0.04539200166861216
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,1,float16,fp8,131071,0.0460746685663859
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,1,float16,float16,1,0.011685332904259363
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,1,float16,fp8,1,0.012282667060693106
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,1,float16,float16,3,0.011253333340088526
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,1,float16,fp8,3,0.012058666596810022
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,1,float16,fp8,7,0.012389333297808966
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,1,float16,float16,7,0.011610666910807291
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,1,float16,float16,15,0.012080000092585882
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,1,float16,fp8,15,0.012234666695197424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,1,float16,float16,31,0.010863999525705973
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,1,float16,fp8,31,0.011930666863918304
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,1,float16,float16,63,0.011978667229413986
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,1,float16,fp8,63,0.011802667131026586
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,1,float16,float16,127,0.01240533341964086
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,1,float16,fp8,127,0.013061333447694778
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,1,float16,float16,255,0.012159999459981918
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,1,float16,fp8,255,0.012938667088747025
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,1,float16,float16,511,0.013487999637921652
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,1,float16,fp8,511,0.012389333297808966
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,1,float16,float16,1023,0.014042666802803675
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,1,float16,fp8,1023,0.013653332988421122
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,1,float16,float16,2047,0.012560000022252401
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,1,float16,fp8,2047,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,1,float16,float16,4095,0.013482666263977686
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,1,float16,fp8,4095,0.013482666263977686
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,1,float16,float16,8191,0.018661333868900936
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,1,float16,fp8,8191,0.01859733338157336
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,1,float16,float16,16383,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,1,float16,fp8,16383,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,1,float16,float16,32767,0.028336000939210255
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,1,float16,fp8,32767,0.02696000039577484
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,1,float16,float16,65535,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,1,float16,fp8,65535,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,1,float16,float16,131071,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,1,float16,float16,1,0.012709333250919977
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,1,float16,fp8,131071,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,1,float16,fp8,1,0.011882666498422623
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,1,float16,float16,3,0.012666666259368261
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,1,float16,fp8,3,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,1,float16,float16,7,0.010687999427318573
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,1,float16,fp8,7,0.012442667037248611
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,1,float16,fp8,15,0.011776000261306763
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,1,float16,float16,31,0.012335999558369318
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,1,float16,fp8,31,0.011034666250149408
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,1,float16,float16,63,0.012618667135636011
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,1,float16,fp8,63,0.011029332876205444
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,1,float16,float16,127,0.012400000045696894
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,1,float16,fp8,127,0.010858666151762009
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,1,float16,float16,255,0.01219733307758967
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,1,float16,fp8,255,0.010703999549150467
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,1,float16,float16,511,0.012330666184425354
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,1,float16,fp8,511,0.010533332824707031
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,1,float16,fp8,1023,0.01108266661564509
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,1,float16,float16,1023,0.012400000045696894
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,1,float16,float16,2047,0.012389333297808966
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,1,float16,fp8,2047,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,1,float16,float16,4095,0.012629333883523941
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,1,float16,fp8,4095,0.012560000022252401
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,1,float16,float16,8191,0.018698666244745255
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,1,float16,fp8,8191,0.01904533306757609
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,1,float16,float16,16383,0.022874665757020313
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,1,float16,fp8,16383,0.02252800017595291
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,1,float16,float16,32767,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,1,float16,fp8,32767,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,1,float16,float16,65535,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,1,float16,float16,131071,0.043354665239652
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,1,float16,fp8,131071,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,1,float16,float16,1,0.010522666076819101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,1,float16,fp8,1,0.01051733394463857
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,1,float16,float16,3,0.011706666400035223
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,1,float16,fp8,3,0.010682666053374609
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,1,float16,float16,7,0.01051733394463857
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,1,float16,fp8,7,0.010853332777818045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,1,float16,float16,15,0.011029332876205444
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,1,float16,fp8,15,0.011178666104873022
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,1,float16,float16,31,0.010672000547250112
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,1,float16,fp8,1023,0.014565333724021912
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,1,float16,fp8,31,0.010853332777818045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,1,float16,float16,63,0.012335999558369318
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,1,float16,fp8,63,0.010490667074918747
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,1,float16,fp8,127,0.010853332777818045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,1,float16,float16,127,0.010816000401973724
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,1,float16,float16,255,0.010853332777818045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,1,float16,fp8,255,0.01349866638580958
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,1,float16,float16,511,0.012085333466529846
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,1,float16,fp8,511,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,1,float16,float16,1023,0.012389333297808966
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,1,float16,fp8,1023,0.01268799975514412
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,1,float16,float16,2047,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,1,float16,fp8,2047,0.012682666381200155
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,1,float16,float16,4095,0.01670933390657107
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,1,float16,fp8,4095,0.016682667036851246
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,1,float16,float16,8191,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,1,float16,fp8,8191,0.018858666221300762
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,1,float16,float16,16383,0.021162666380405426
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,1,float16,fp8,16383,0.02082666630546252
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,1,float16,float16,32767,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,1,float16,fp8,32767,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,1,float16,float16,65535,0.03788800040880839
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,1,float16,fp8,65535,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,1,float16,float16,131071,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,1,float16,fp8,131071,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,1,float16,float16,1,0.010687999427318573
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,1,float16,fp8,1,0.012800000607967377
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,1,float16,float16,3,0.01051733394463857
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,1,float16,fp8,3,0.012442667037248611
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,1,float16,float16,7,0.012223999947309494
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,1,float16,fp8,7,0.01251199965675672
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,1,float16,float16,15,0.010821333775917688
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,1,float16,fp8,15,0.012181332955757776
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,1,float16,float16,31,0.011029332876205444
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,1,float16,fp8,31,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,1,float16,float16,63,0.010858666151762009
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,1,float16,fp8,63,0.013728000223636627
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,1,float16,float16,127,0.010863999525705973
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,1,float16,fp8,127,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,1,float16,float16,255,0.010863999525705973
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,1,float16,fp8,255,0.012789333860079447
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,1,float16,float16,511,0.011760000139474869
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,1,float16,fp8,511,0.013034666577974955
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,1,float16,float16,1023,0.010879999647537867
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,1,float16,fp8,1023,0.012863999853531519
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,1,float16,float16,2047,0.012629333883523941
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,1,float16,fp8,2047,0.014570667097965876
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,1,float16,float16,4095,0.014335999886194864
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,1,float16,fp8,4095,0.014560000350077948
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,1,float16,float16,8191,0.02075200031201045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,1,float16,fp8,8191,0.020725333442290623
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,1,float16,float16,16383,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,1,float16,fp8,16383,0.02696000039577484
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,1,float16,float16,32767,0.029365333418051403
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,1,float16,fp8,32767,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,1,float16,float16,65535,0.032085334261258446
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,1,float16,fp8,65535,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,1,float16,float16,131071,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,1,float16,fp8,131071,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,1,float16,float16,1,0.010858666151762009
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,1,float16,fp8,1,0.01394133393963178
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,1,float16,float16,3,0.011424000064531961
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,1,float16,fp8,3,0.012608000387748083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,1,float16,float16,7,0.01051733394463857
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,1,float16,fp8,7,0.01251199965675672
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,1,float16,float16,15,0.011098666737476984
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,1,float16,fp8,15,0.012629333883523941
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,1,float16,float16,31,0.011258666714032492
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,1,float16,fp8,31,0.014138666292031607
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,1,float16,float16,63,0.011034666250149408
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,1,float16,fp8,63,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,1,float16,float16,127,0.011930666863918304
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,1,float16,fp8,127,0.012863999853531519
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,1,float16,float16,255,0.012453333785136541
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,1,float16,fp8,255,0.014175999909639359
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,1,float16,float16,511,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,1,float16,fp8,511,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,1,float16,float16,1023,0.012378666549921036
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,1,float16,fp8,1023,0.01320533330241839
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,1,float16,float16,2047,0.01258133351802826
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,1,float16,fp8,2047,0.012693333129088083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,1,float16,float16,4095,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,1,float16,fp8,4095,0.014570667097965876
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,1,float16,float16,8191,0.019296000401178997
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,1,float16,fp8,8191,0.02070933332045873
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,1,float16,float16,16383,0.022863999009132385
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,1,float16,fp8,16383,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,1,float16,float16,32767,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,1,float16,fp8,32767,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,1,float16,float16,65535,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,1,float16,fp8,65535,0.025258667767047882
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,1,float16,float16,131071,0.04539733131726583
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,1,float16,fp8,131071,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,1,float16,float16,1,0.01119999960064888
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,1,float16,fp8,1,0.012479999413092932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,1,float16,float16,3,0.011173332730929056
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,1,float16,fp8,3,0.011258666714032492
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,1,float16,float16,7,0.010842667271693548
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,1,float16,fp8,7,0.014362666755914688
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,1,float16,float16,15,0.011264000087976456
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,1,float16,fp8,15,0.012373333175977072
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,1,float16,float16,31,0.011866666376590729
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,1,float16,fp8,31,0.012458667159080505
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,1,float16,float16,63,0.01145600030819575
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,1,float16,fp8,63,0.012549333274364471
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,1,float16,float16,127,0.011434666812419891
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,1,float16,fp8,127,0.012709333250919977
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,1,float16,float16,255,0.011674666156371435
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,1,float16,fp8,255,0.012015999605258306
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,1,float16,float16,511,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,1,float16,fp8,511,0.012346666306257248
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,1,float16,float16,1023,0.012560000022252401
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,1,float16,fp8,1023,0.012863999853531519
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,1,float16,float16,2047,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,1,float16,fp8,2047,0.014570667097965876
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,1,float16,float16,4095,0.016554666062196095
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,1,float16,fp8,4095,0.01682666689157486
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,1,float16,float16,8191,0.01904533306757609
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,1,float16,fp8,8191,0.01899733394384384
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,1,float16,float16,16383,0.02109866589307785
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,1,float16,fp8,16383,0.021029333273569744
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,1,float16,float16,32767,0.024911999702453613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,1,float16,fp8,32767,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,1,float16,float16,65535,0.03754666695992152
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,1,float16,fp8,65535,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,1,float16,float16,131071,0.062122667829195656
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,1,float16,fp8,131071,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,1,float16,float16,1,0.010581333190202713
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,1,float16,fp8,1,0.011434666812419891
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,1,float16,float16,3,0.011034666250149408
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,1,float16,fp8,3,0.012069333344697952
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,1,float16,float16,7,0.011264000087976456
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,1,float16,fp8,7,0.012341332932313284
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,1,float16,float16,15,0.011829332758982977
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,1,float16,fp8,15,0.012965332716703415
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,1,float16,float16,31,0.012037333101034164
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,1,float16,fp8,31,0.012378666549921036
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,1,float16,float16,63,0.012058666596810022
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,1,float16,fp8,63,0.012800000607967377
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,1,float16,float16,127,0.011605333536863327
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,1,float16,fp8,127,0.012096000214417776
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,1,float16,float16,255,0.011434666812419891
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,1,float16,fp8,255,0.011989332735538483
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,1,float16,float16,511,0.011653333902359009
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,1,float16,fp8,511,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,1,float16,float16,1023,0.012400000045696894
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,1,float16,fp8,1023,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,1,float16,float16,2047,0.014677333335081736
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,1,float16,fp8,2047,0.016469333320856094
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,1,float16,float16,4095,0.016677333662907284
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,1,float16,fp8,4095,0.01695999999841054
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,1,float16,float16,8191,0.018709332992633183
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,1,float16,fp8,8191,0.019007999449968338
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,1,float16,float16,16383,0.022874665757020313
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,1,float16,fp8,16383,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,1,float16,float16,32767,0.03824000060558319
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,1,float16,fp8,32767,0.024906667570273083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,1,float16,float16,65535,0.0580320010582606
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,1,float16,fp8,65535,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,1,float16,float16,131071,0.09864532947540283
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,1,float16,fp8,131071,0.059392000238100685
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,1,float16,float16,1,0.011029332876205444
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,1,float16,fp8,1,0.012693333129088083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,1,float16,float16,3,0.011434666812419891
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,1,float16,fp8,3,0.013770667215188345
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,1,float16,float16,7,0.012560000022252401
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,1,float16,fp8,7,0.01340266689658165
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,1,float16,float16,15,0.01109333336353302
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,1,float16,fp8,15,0.012794667234023413
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,1,float16,float16,31,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,1,float16,fp8,31,0.01268799975514412
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,1,float16,float16,63,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,1,float16,fp8,63,0.012682666381200155
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,1,float16,float16,127,0.012629333883523941
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,1,float16,fp8,127,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,1,float16,float16,255,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,1,float16,fp8,255,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,1,float16,float16,511,0.012901333471139273
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,1,float16,fp8,511,0.01461333284775416
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,1,float16,float16,1023,0.014511999984582266
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,1,float16,fp8,1023,0.014570667097965876
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,1,float16,float16,2047,0.018709332992633183
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,1,float16,fp8,2047,0.018672000616788864
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,1,float16,float16,4095,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,1,float16,fp8,4095,0.02252800017595291
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,1,float16,float16,8191,0.040949332217375435
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,1,float16,fp8,8191,0.030026666820049286
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,1,float16,float16,16383,0.06144000093142191
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,1,float16,fp8,16383,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,1,float16,float16,1,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,1,float16,fp8,1,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,1,float16,float16,3,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,1,float16,fp8,3,0.013760000467300415
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,1,float16,float16,7,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,1,float16,fp8,7,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,1,float16,float16,15,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,1,float16,fp8,15,0.01251199965675672
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,1,float16,float16,31,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,1,float16,fp8,31,0.01251199965675672
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,1,float16,float16,63,0.01239466667175293
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,1,float16,fp8,63,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,1,float16,float16,127,0.011205332974592844
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,1,float16,fp8,127,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,1,float16,float16,255,0.01210133358836174
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,1,float16,fp8,255,0.01350933313369751
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,1,float16,float16,511,0.012879999975363413
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,1,float16,fp8,511,0.01302933320403099
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,1,float16,float16,1023,0.014511999984582266
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,1,float16,fp8,1023,0.014576000471909841
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,1,float16,float16,2047,0.022805333137512207
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,1,float16,fp8,2047,0.020714666694402695
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,1,float16,float16,4095,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,1,float16,fp8,4095,0.02696000039577484
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,1,float16,float16,8191,0.06144000093142191
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,1,float16,fp8,8191,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,1,float16,float16,16383,0.10239467024803162
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,1,float16,fp8,16383,0.06178666651248932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,1,float16,fp8,1,0.014741333822409311
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,1,float16,float16,1,0.014607999473810196
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,1,float16,float16,3,0.014565333724021912
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,1,float16,fp8,3,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,1,float16,float16,7,0.014736000448465347
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,1,float16,fp8,7,0.012714666624863943
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,1,float16,float16,15,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,1,float16,fp8,15,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,1,float16,float16,15,0.010650667051474253
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,1,float16,float16,31,0.014570667097965876
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,1,float16,fp8,31,0.0129120002190272
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,1,float16,float16,63,0.014565333724021912
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,1,float16,fp8,63,0.015189333508412043
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,1,float16,float16,127,0.0129120002190272
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,1,float16,fp8,127,0.01440000037352244
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,1,float16,float16,255,0.014570667097965876
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,1,float16,fp8,255,0.012730666746695837
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,1,float16,float16,511,0.016719999412695568
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,1,float16,fp8,511,0.016613333175579708
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,1,float16,float16,1023,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,1,float16,fp8,1023,0.01851733277241389
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,1,float16,float16,2047,0.04028266668319702
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,1,float16,fp8,2047,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,1,float16,float16,4095,0.06177600224812826
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,1,float16,fp8,4095,0.041296000281969704
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,1,float16,float16,8191,0.10342400272687276
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,1,float16,fp8,8191,0.06382933259010315
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,1,float16,float16,16383,0.18687466780344644
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,1,float16,fp8,16383,0.10513066252072652
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,1,float16,float16,1,0.016607999801635742
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,1,float16,fp8,1,0.014896000425020853
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,1,float16,float16,3,0.016672000288963318
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,1,float16,fp8,3,0.014917333920796713
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,1,float16,float16,7,0.01695466662446658
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,1,float16,fp8,7,0.014511999984582266
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,1,float16,float16,15,0.01695999999841054
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,1,float16,fp8,15,0.014741333822409311
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,1,float16,float16,31,0.016656000167131424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,1,float16,fp8,31,0.014912000546852747
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,1,float16,float16,63,0.016805333395799
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,1,float16,fp8,63,0.014842666685581207
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,1,float16,fp8,65535,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,1,float16,float16,127,0.016805333395799
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,1,float16,fp8,127,0.014912000546852747
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,1,float16,float16,255,0.016656000167131424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,1,float16,fp8,255,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,1,float16,float16,511,0.021242665747801464
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,1,float16,fp8,511,0.018863999595244724
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,1,float16,float16,1023,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,1,float16,fp8,1023,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,1,float16,float16,2047,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,1,float16,fp8,2047,0.043007999658584595
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,1,float16,float16,4095,0.10478933652242024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,1,float16,fp8,4095,0.06382933259010315
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,1,float16,float16,8191,0.1889280080795288
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,1,float16,fp8,8191,0.10683733224868774
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,1,float16,float16,16383,0.3585653305053711
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,1,float16,float16,1,0.012186666329701742
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,1,float16,fp8,1,0.012741333494583765
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,1,float16,float16,3,0.011978667229413986
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,1,float16,fp8,3,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,1,float16,float16,7,0.013823999712864557
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,1,float16,fp8,7,0.01239466667175293
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,1,float16,float16,15,0.010816000401973724
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,1,float16,fp8,15,0.012293333808581034
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,1,float16,float16,31,0.011770666887362799
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,1,float16,fp8,31,0.01166933278242747
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,1,float16,float16,63,0.012346666306257248
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,1,float16,fp8,63,0.011893333246310553
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,1,float16,float16,127,0.012693333129088083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,1,float16,fp8,127,0.01227733368674914
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,1,float16,float16,255,0.011855999628702799
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,1,float16,fp8,255,0.01099733387430509
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,1,float16,float16,511,0.01267733300725619
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,1,float16,fp8,511,0.012560000022252401
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,1,float16,float16,1023,0.012693333129088083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,1,float16,fp8,1023,0.01302933320403099
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,1,float16,float16,2047,0.014677333335081736
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,1,float16,fp8,2047,0.014752000570297241
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,1,float16,float16,4095,0.01646399994691213
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,1,float16,fp8,4095,0.016672000288963318
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,1,float16,float16,8191,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,1,float16,fp8,8191,0.022874665757020313
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,1,float16,float16,16383,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,1,float16,fp8,16383,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,1,float16,float16,32767,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,1,float16,fp8,32767,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,1,float16,float16,65535,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,1,float16,fp8,65535,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,1,float16,float16,131071,0.04949333270390829
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,1,float16,float16,1,0.01268799975514412
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,1,float16,fp8,131071,0.035829332967599235
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,1,float16,fp8,1,0.012383999923865
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,1,float16,float16,3,0.012543999900420507
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,1,float16,fp8,3,0.012096000214417776
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,1,float16,float16,7,0.011829332758982977
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,1,float16,fp8,7,0.011253333340088526
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,1,float16,float16,15,0.011823999385039011
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,1,float16,fp8,15,0.016976000120242436
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,1,float16,float16,31,0.01210133358836174
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,1,float16,fp8,31,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,1,float16,float16,63,0.011946666985750198
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,1,float16,fp8,63,0.012671999633312225
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,1,float16,float16,127,0.011584000041087469
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,1,float16,fp8,127,0.013173333058754602
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,1,float16,float16,255,0.012778667112191519
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,1,float16,fp8,255,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,1,float16,float16,511,0.011087999989589056
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,1,float16,fp8,511,0.012965332716703415
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,1,float16,float16,1023,0.012863999853531519
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,1,float16,fp8,1023,0.012400000045696894
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,1,float16,float16,2047,0.014570667097965876
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,1,float16,float16,4095,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,1,float16,fp8,4095,0.01884799947341283
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,1,float16,float16,8191,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,1,float16,fp8,8191,0.020714666694402695
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,1,float16,float16,16383,0.022805333137512207
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,1,float16,fp8,16383,0.022863999009132385
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,1,float16,float16,32767,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,1,float16,fp8,32767,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,1,float16,float16,65535,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,1,float16,fp8,65535,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,1,float16,float16,131071,0.0628053347269694
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,1,float16,float16,1,0.010512000570694605
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,1,float16,fp8,1,0.011546666423479715
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,1,float16,float16,3,0.010512000570694605
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,1,float16,fp8,3,0.012229333321253458
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,1,float16,float16,7,0.010522666076819101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,1,float16,fp8,7,0.010933333386977514
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,1,float16,float16,15,0.01051733394463857
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,1,float16,fp8,15,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,1,float16,float16,31,0.010682666053374609
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,1,float16,fp8,31,0.010522666076819101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,1,float16,float16,63,0.010677333921194077
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,1,float16,fp8,63,0.011882666498422623
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,1,float16,float16,127,0.010512000570694605
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,1,float16,fp8,127,0.011546666423479715
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,1,float16,float16,255,0.010693332801262537
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,1,float16,fp8,255,0.011264000087976456
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,1,float16,float16,511,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,1,float16,fp8,511,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,1,float16,float16,1023,0.012357333054145178
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,1,float16,fp8,1023,0.012400000045696894
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,1,float16,float16,2047,0.01666133354107539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,1,float16,fp8,2047,0.016672000288963318
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,1,float16,float16,4095,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,1,float16,fp8,4095,0.01865600049495697
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,1,float16,float16,8191,0.0206986665725708
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,1,float16,fp8,8191,0.019754666835069656
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,1,float16,float16,16383,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,1,float16,fp8,16383,0.022874665757020313
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,1,float16,float16,32767,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,1,float16,fp8,32767,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,1,float16,float16,65535,0.059392000238100685
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,1,float16,fp8,65535,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,1,float16,float16,131071,0.09967466195424397
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,1,float16,fp8,131071,0.06144000093142191
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,1,float16,float16,1,0.010522666076819101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,1,float16,fp8,1,0.010853332777818045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,1,float16,float16,3,0.013056000073750814
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,1,float16,fp8,3,0.01109333336353302
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,1,float16,float16,7,0.011429333438475927
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,1,float16,fp8,7,0.010687999427318573
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,1,float16,float16,15,0.010741333166758219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,1,float16,fp8,15,0.010703999549150467
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,1,float16,float16,31,0.010858666151762009
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,1,float16,fp8,31,0.011039999624093374
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,1,float16,float16,63,0.010512000570694605
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,1,float16,fp8,63,0.010853332777818045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,1,float16,float16,127,0.010853332777818045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,1,float16,fp8,127,0.01102399950226148
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,1,float16,float16,255,0.012335999558369318
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,1,float16,fp8,255,0.010863999525705973
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,1,float16,float16,511,0.011744000017642975
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,1,float16,fp8,511,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,1,float16,float16,1023,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,1,float16,fp8,1023,0.012709333250919977
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,1,float16,float16,2047,0.016666666915019352
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,1,float16,fp8,2047,0.01551466683546702
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,1,float16,float16,4095,0.01870399961868922
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,1,float16,fp8,4095,0.01672533278663953
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,1,float16,float16,8191,0.022533332308133442
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,1,float16,fp8,8191,0.018719999740521114
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,1,float16,float16,16383,0.03721066564321518
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,1,float16,fp8,16383,0.02492266645034154
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,1,float16,float16,32767,0.05870933334032694
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,1,float16,fp8,32767,0.03721600025892258
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,1,float16,float16,65535,0.10068800052007039
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,1,float16,fp8,65535,0.0576853354771932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,1,float16,float16,131071,0.18619734048843384
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,1,float16,fp8,131071,0.10058666268984477
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,1,float16,float16,1,0.012629333883523941
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,1,float16,fp8,1,0.012458667159080505
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,1,float16,float16,3,0.01246400053302447
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,1,float16,fp8,3,0.012453333785136541
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,1,float16,float16,7,0.01246400053302447
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,1,float16,fp8,7,0.012725333372751871
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,1,float16,float16,15,0.012554666648308435
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,1,float16,fp8,15,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,1,float16,float16,31,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,1,float16,fp8,31,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,1,float16,float16,63,0.012698666503032049
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,1,float16,fp8,63,0.012357333054145178
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,1,float16,float16,127,0.012560000022252401
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,1,float16,fp8,127,0.012453333785136541
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,1,float16,float16,255,0.012458667159080505
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,1,float16,fp8,255,0.012458667159080505
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,1,float16,float16,511,0.014511999984582266
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,1,float16,fp8,511,0.012629333883523941
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,1,float16,float16,1023,0.016656000167131424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,1,float16,fp8,1023,0.014629332969586054
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,1,float16,float16,2047,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,1,float16,fp8,2047,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,1,float16,float16,4095,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,1,float16,fp8,4095,0.027984000742435455
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,1,float16,float16,8191,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,1,float16,fp8,8191,0.04369066655635834
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,1,float16,float16,1,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,1,float16,fp8,1,0.012810666114091873
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,1,float16,float16,3,0.014607999473810196
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,1,float16,fp8,3,0.012714666624863943
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,1,float16,float16,7,0.0145066666106383
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,1,float16,fp8,7,0.012543999900420507
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,1,float16,float16,15,0.012805332740147909
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,1,float16,fp8,15,0.012794667234023413
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,1,float16,float16,31,0.013248000293970108
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,1,float16,fp8,31,0.0129120002190272
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,1,float16,float16,63,0.014511999984582266
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,1,float16,fp8,63,0.012874666601419449
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,1,float16,float16,127,0.01441066712141037
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,1,float16,fp8,127,0.012906666845083237
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,1,float16,float16,255,0.014501333236694336
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,1,float16,fp8,255,0.012736000120639801
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,1,float16,float16,511,0.016656000167131424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,1,float16,fp8,511,0.014607999473810196
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,1,float16,float16,1023,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,1,float16,fp8,1023,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,1,float16,float16,2047,0.03993066648642222
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,1,float16,fp8,2047,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,1,float16,float16,4095,0.062458669145902
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,1,float16,fp8,4095,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,1,float16,float16,8191,0.10444800059000652
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,1,float16,fp8,8191,0.06348800162474315
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,1,float16,float16,1,0.016666666915019352
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,1,float16,fp8,1,0.014959999670584997
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,1,float16,float16,3,0.01653333380818367
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,1,float16,fp8,3,0.014848000059525171
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,1,float16,float16,7,0.016666666915019352
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,1,float16,fp8,7,0.014762666076421738
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,1,float16,fp8,16383,0.19166400035222372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,1,float16,float16,15,0.01666133354107539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,1,float16,fp8,15,0.01461333284775416
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,1,float16,float16,31,0.01681600014368693
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,1,float16,fp8,31,0.014959999670584997
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,1,float16,float16,63,0.016650666793187458
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,1,float16,fp8,63,0.01462399959564209
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,1,float16,float16,127,0.016544000556071598
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,1,float16,fp8,127,0.014949332922697067
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,1,float16,float16,255,0.016656000167131424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,1,float16,fp8,255,0.014943999548753103
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,1,float16,float16,511,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,1,float16,fp8,511,0.01859733338157336
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,1,float16,float16,1023,0.03345600018898646
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,1,float16,fp8,1023,0.024847999215126038
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,1,float16,float16,2047,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,1,float16,fp8,2047,0.041296000281969704
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,1,float16,float16,4095,0.10513066252072652
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,1,float16,fp8,4095,0.06382933259010315
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,1,float16,float16,8191,0.18841065963109335
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,1,float16,fp8,8191,0.10718400279680888
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,1,float16,float16,1,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,1,float16,fp8,1,0.01884799947341283
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,1,float16,float16,3,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,1,float16,fp8,3,0.018709332992633183
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,1,float16,float16,7,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,1,float16,fp8,7,0.018944000204404194
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,1,float16,float16,15,0.021168000996112823
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,1,float16,fp8,15,0.018954666952292126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,1,float16,float16,31,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,1,float16,fp8,31,0.018709332992633183
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,1,float16,float16,63,0.022863999009132385
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,1,float16,fp8,63,0.018709332992633183
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,1,float16,float16,127,0.02081599955757459
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,1,float16,fp8,127,0.018709332992633183
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,1,float16,float16,255,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,1,float16,fp8,255,0.018863999595244724
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,1,float16,float16,511,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,1,float16,fp8,511,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,1,float16,fp8,2047,0.014570667097965876
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,1,float16,float16,1023,0.05394133428732554
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,1,float16,fp8,1023,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,1,float16,float16,2047,0.09796266754468282
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,1,float16,fp8,2047,0.060415998101234436
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,1,float16,float16,4095,0.17476266622543335
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,1,float16,fp8,4095,0.09864532947540283
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,1,float16,float16,8191,0.33126399914423627
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,1,float16,float16,1,0.016629333297411602
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,1,float16,fp8,8191,0.17323732376098633
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,1,float16,fp8,1,0.014602666099866232
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,1,float16,float16,3,0.01525866612792015
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,1,float16,fp8,3,0.01461333284775416
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,1,float16,float16,7,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,1,float16,fp8,7,0.014560000350077948
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,1,float16,float16,15,0.014794666320085526
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,1,float16,fp8,15,0.01451733335852623
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,1,float16,fp8,131071,0.04505600035190582
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,1,float16,float16,31,0.01611199975013733
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,1,float16,fp8,31,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,1,float16,float16,63,0.014949332922697067
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,1,float16,fp8,63,0.01423466702302297
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,1,float16,float16,127,0.015018666783968607
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,1,float16,fp8,127,0.014607999473810196
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,1,float16,float16,255,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,1,float16,fp8,255,0.014576000471909841
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,1,float16,float16,511,0.017301333447297413
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,1,float16,fp8,511,0.016554666062196095
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,1,float16,float16,1023,0.022437334060668945
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,1,float16,fp8,1023,0.02070933332045873
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,1,float16,float16,2047,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,1,float16,fp8,2047,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,1,float16,float16,1,0.016997333616018295
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,1,float16,fp8,1,0.016789333273967106
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,1,float16,float16,3,0.01798933371901512
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,1,float16,fp8,3,0.01461333284775416
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,1,float16,float16,7,0.016800000021855038
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,1,float16,fp8,7,0.01661866654952367
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,1,float16,float16,15,0.0183146670460701
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,1,float16,fp8,15,0.016656000167131424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,1,float16,float16,31,0.01700266698996226
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,1,float16,fp8,31,0.016607999801635742
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,1,float16,float16,63,0.01695466662446658
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,1,float16,fp8,63,0.01461333284775416
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,1,float16,float16,127,0.017008000363906223
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,1,float16,fp8,127,0.016672000288963318
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,1,float16,float16,255,0.016794666647911072
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,1,float16,fp8,255,0.01666133354107539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,1,float16,float16,511,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,1,float16,fp8,511,0.018677332748969395
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,1,float16,float16,1023,0.035829332967599235
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,1,float16,fp8,1023,0.024853333830833435
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,1,float16,float16,2047,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,1,float16,fp8,2047,0.04232533276081085
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,1,float16,float16,1,0.022863999009132385
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,1,float16,fp8,1,0.02041600023706754
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,1,float16,float16,3,0.022863999009132385
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,1,float16,fp8,3,0.02070933332045873
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,1,float16,float16,7,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,1,float16,fp8,7,0.020410666863123577
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,1,float16,float16,15,0.02279466638962428
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,1,float16,fp8,15,0.018863999595244724
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,1,float16,fp8,31,0.019050666441520054
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,1,float16,float16,31,0.022757334013779957
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,1,float16,float16,63,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,1,float16,fp8,63,0.020714666694402695
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,1,float16,float16,127,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,1,float16,fp8,127,0.01904533306757609
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,1,float16,float16,255,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,1,float16,fp8,255,0.020714666694402695
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,1,float16,float16,511,0.034474665919939675
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,1,float16,fp8,511,0.02515200028816859
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,1,float16,float16,1023,0.05563200016816457
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,1,float16,fp8,1023,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,1,float16,float16,2047,0.09830400347709656
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,1,float16,fp8,2047,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,1,float16,float16,1,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,1,float16,fp8,1,0.02867199977238973
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,1,float16,float16,3,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,1,float16,fp8,3,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,1,float16,float16,7,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,1,float16,fp8,7,0.028336000939210255
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,1,float16,float16,15,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,1,float16,fp8,15,0.027130665878454845
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,1,float16,float16,31,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,1,float16,fp8,31,0.027999999622503918
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,1,float16,float16,63,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,1,float16,fp8,63,0.02867199977238973
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,1,float16,float16,127,0.03345066557327906
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,1,float16,fp8,127,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,1,float16,float16,255,0.03549866626660029
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,1,float16,fp8,255,0.02697066714366277
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,1,float16,float16,511,0.05461333195368449
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,1,float16,fp8,511,0.039605334401130676
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,1,float16,float16,1023,0.09454933802286784
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,1,float16,fp8,1023,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,1,float16,float16,2047,0.17544533809026083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,1,float16,fp8,2047,0.10103467106819153
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,1,float16,float16,1,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,1,float16,fp8,1,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,1,float16,float16,3,0.02075200031201045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,1,float16,fp8,3,0.0186666672428449
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,1,float16,float16,7,0.02067199970285098
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,1,float16,fp8,7,0.018677332748969395
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,1,float16,float16,15,0.02013333390156428
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,1,float16,fp8,15,0.018863999595244724
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,1,float16,float16,31,0.02065066620707512
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,1,float16,fp8,31,0.018661333868900936
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,1,float16,float16,63,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,1,float16,fp8,63,0.01899733394384384
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,1,float16,float16,127,0.020080000162124634
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,1,float16,fp8,127,0.01868266612291336
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,1,float16,float16,255,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,1,float16,fp8,255,0.0186666672428449
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,1,float16,float16,511,0.024234667420387268
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,1,float16,fp8,511,0.02276266614596049
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,1,float16,float16,1023,0.03754133234421412
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,1,float16,fp8,1023,0.026954665780067444
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,1,float16,float16,1,0.022800001005331676
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,1,float16,fp8,1,0.020703999946514767
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,1,float16,float16,3,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,1,float16,float16,7,0.02346666653951009
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,1,float16,fp8,3,0.020410666863123577
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,1,float16,fp8,7,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,1,float16,float16,15,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,1,float16,fp8,15,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,1,float16,float16,31,0.023823998868465424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,1,float16,fp8,31,0.021055998901526134
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,1,float16,float16,63,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,1,float16,fp8,63,0.02072000006834666
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,1,float16,float16,127,0.022805333137512207
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,1,float16,fp8,127,0.020714666694402695
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,1,float16,float16,255,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,1,float16,fp8,255,0.02072000006834666
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,1,float16,float16,511,0.035504000882307686
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,1,float16,fp8,511,0.02697066714366277
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,1,float16,float16,1023,0.05563733478387197
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,1,float16,float16,1,0.034815999368826546
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,1,float16,fp8,1,0.02908266584078471
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,1,float16,fp8,1023,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,1,float16,float16,3,0.035162667433420815
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,1,float16,fp8,3,0.028666667640209198
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,1,float16,float16,7,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,1,float16,fp8,7,0.028688001135985058
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,1,float16,float16,15,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,1,float16,fp8,15,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,1,float16,float16,31,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,1,float16,float16,63,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,1,float16,fp8,63,0.026949333647886913
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,1,float16,float16,127,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,1,float16,fp8,127,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,1,float16,float16,255,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,1,float16,fp8,255,0.029018667836983997
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,1,float16,float16,511,0.05563733478387197
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,1,float16,fp8,511,0.041306667029857635
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,1,float16,float16,1023,0.09591466188430786
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,1,float16,fp8,1023,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,1,float16,float16,1,0.05699733396371206
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,1,float16,fp8,1,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,1,float16,float16,3,0.0576853354771932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,1,float16,fp8,3,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,1,float16,float16,7,0.0576853354771932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,1,float16,fp8,7,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,1,float16,float16,15,0.05701333284378052
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,1,float16,fp8,15,0.043354665239652
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,1,float16,float16,31,0.05563733478387197
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,1,float16,fp8,31,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,1,float16,float16,63,0.0576853354771932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,1,float16,fp8,63,0.043354665239652
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,1,float16,float16,127,0.0576853354771932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,1,float16,fp8,127,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,1,float16,float16,255,0.05973866581916809
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,1,float16,fp8,255,0.04539200166861216
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,1,float16,float16,511,0.09523199995358785
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,1,float16,fp8,511,0.06587733328342438
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,1,float16,float16,1,0.011765333513418833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,1,float16,float16,1023,0.17220266660054526
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,1,float16,fp8,1023,0.10103467106819153
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,1,float16,fp8,1,0.012058666596810022
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,1,float16,float16,3,0.012138667205969492
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,1,float16,fp8,3,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,1,float16,float16,7,0.012634667257467905
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,1,float16,fp8,7,0.01219733307758967
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,1,float16,float16,15,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,1,float16,fp8,15,0.012063999970753988
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,1,float16,float16,31,0.012479999413092932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,1,float16,fp8,31,0.012954667210578918
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,1,float16,float16,63,0.01240533341964086
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,1,float16,fp8,63,0.011600000162919363
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,1,float16,float16,127,0.01091733326514562
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,1,float16,fp8,127,0.012053333222866058
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,1,float16,fp8,255,0.012863999853531519
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,1,float16,float16,511,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,1,float16,fp8,511,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,1,float16,float16,1023,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,1,float16,fp8,1023,0.01258133351802826
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,1,float16,fp8,2047,0.014677333335081736
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,1,float16,float16,2047,0.014607999473810196
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,1,float16,float16,4095,0.019013332823912304
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,1,float16,fp8,4095,0.01870399961868922
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,1,float16,float16,8191,0.021498667697111767
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,1,float16,fp8,8191,0.02117866774400075
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,1,float16,float16,16383,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,1,float16,fp8,16383,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,1,float16,float16,32767,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,1,float16,fp8,32767,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,1,float16,float16,65535,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,1,float16,fp8,65535,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,1,float16,float16,131071,0.06587733328342438
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,1,float16,fp8,131071,0.04744000236193339
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,1,float16,float16,1,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,1,float16,fp8,1,0.011018666128317514
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,1,float16,float16,3,0.012069333344697952
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,1,float16,fp8,3,0.010682666053374609
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,1,float16,float16,7,0.011440000186363855
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,1,float16,fp8,7,0.010863999525705973
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,1,float16,float16,15,0.011999999483426413
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,1,float16,fp8,15,0.010853332777818045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,1,float16,float16,31,0.011792000383138657
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,1,float16,fp8,31,0.011264000087976456
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,1,float16,float16,63,0.01081066702802976
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,1,float16,fp8,63,0.010847999403874079
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,1,float16,float16,127,0.011877333124478659
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,1,float16,fp8,127,0.010682666053374609
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,1,float16,float16,255,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,1,float16,fp8,255,0.010863999525705973
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,1,float16,float16,511,0.012863999853531519
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,1,float16,fp8,511,0.012378666549921036
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,1,float16,float16,1023,0.01268799975514412
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,1,float16,fp8,1023,0.012560000022252401
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,1,float16,float16,2047,0.017024000485738117
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,1,float16,fp8,2047,0.016656000167131424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,1,float16,fp8,4095,0.018432000031073887
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,1,float16,float16,4095,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,1,float16,float16,8191,0.02081599955757459
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,1,float16,fp8,8191,0.02070933332045873
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,1,float16,float16,16383,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,1,float16,fp8,16383,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,1,float16,fp8,32767,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,1,float16,float16,65535,0.06109866499900818
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,1,float16,fp8,65535,0.040965333580970764
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,1,float16,float16,131071,0.10171733299891154
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,1,float16,fp8,131071,0.0628053347269694
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,1,float16,float16,1,0.011157333850860596
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,1,float16,fp8,1,0.012351999680201212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,1,float16,float16,3,0.010821333775917688
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,1,float16,fp8,3,0.011296000331640244
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,1,float16,float16,7,0.01110400011142095
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,1,float16,fp8,7,0.011952000359694162
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,1,float16,float16,15,0.010965333630641302
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,1,float16,fp8,15,0.011168000598748526
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,1,float16,float16,31,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,1,float16,fp8,31,0.012351999680201212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,1,float16,float16,63,0.012442667037248611
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,1,float16,fp8,63,0.012351999680201212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,1,float16,float16,127,0.012576000144084295
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,1,float16,fp8,127,0.011328000575304031
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,1,float16,float16,255,0.011253333340088526
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,1,float16,fp8,255,0.012784000486135483
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,1,float16,float16,511,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,1,float16,fp8,511,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,1,float16,float16,1023,0.012853333105643591
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,1,float16,fp8,1023,0.013183999806642532
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,1,float16,float16,2047,0.016666666915019352
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,1,float16,fp8,2047,0.016469333320856094
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,1,float16,float16,4095,0.017994667092959087
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,1,float16,fp8,4095,0.01666133354107539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,1,float16,float16,8191,0.020997333029905956
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,1,float16,fp8,8191,0.019018666197856266
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,1,float16,float16,16383,0.03857066730658213
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,1,float16,fp8,16383,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,1,float16,float16,32767,0.060080001751581825
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,1,float16,fp8,32767,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,1,float16,float16,65535,0.10069333513577779
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,1,float16,fp8,65535,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,1,float16,float16,131071,0.18449066082636514
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,1,float16,fp8,131071,0.10205866893132527
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,1,float16,float16,1,0.013066666821638743
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,1,float16,fp8,1,0.012282667060693106
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,1,float16,float16,3,0.012015999605258306
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,1,float16,fp8,3,0.01228800043463707
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,1,float16,float16,7,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,1,float16,fp8,7,0.013141332815090815
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,1,float16,float16,15,0.012426666915416718
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,1,float16,fp8,15,0.012351999680201212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,1,float16,float16,31,0.011898666620254517
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,1,float16,fp8,31,0.012810666114091873
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,1,float16,float16,63,0.012106666962305704
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,1,float16,fp8,63,0.012175999581813812
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,1,float16,float16,127,0.012800000607967377
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,1,float16,fp8,127,0.011952000359694162
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,1,float16,float16,255,0.014111999422311783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,1,float16,fp8,255,0.013674666484196981
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,1,float16,float16,511,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,1,float16,fp8,511,0.012863999853531519
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,1,float16,float16,1023,0.012863999853531519
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,1,float16,fp8,1023,0.012863999853531519
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,1,float16,float16,2047,0.018677332748969395
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,1,float16,fp8,2047,0.01695999999841054
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,1,float16,float16,4095,0.021685334543387096
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,1,float16,fp8,4095,0.020714666694402695
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,1,float16,float16,8191,0.03822399924198786
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,1,float16,fp8,8191,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,1,float16,float16,16383,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,1,float16,fp8,16383,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,1,float16,float16,32767,0.10103467106819153
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,1,float16,fp8,32767,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,1,float16,float16,65535,0.18381865819295248
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,1,float16,fp8,65535,0.10034666458765666
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,1,float16,float16,131071,0.35174401601155597
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,1,float16,fp8,131071,0.18191999197006226
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,1,float16,float16,1,0.02900800108909607
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,1,float16,fp8,1,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,1,float16,float16,3,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,1,float16,fp8,3,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,1,float16,float16,7,0.029018667836983997
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,1,float16,fp8,7,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,1,float16,float16,15,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,1,float16,fp8,15,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,1,float16,float16,31,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,1,float16,fp8,31,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,1,float16,float16,63,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,1,float16,fp8,63,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,1,float16,float16,127,0.029018667836983997
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,1,float16,fp8,127,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,1,float16,float16,255,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,1,float16,fp8,255,0.025258667767047882
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,1,float16,float16,511,0.039264000952243805
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,1,float16,fp8,511,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,1,float16,float16,1,0.0365226666132609
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,1,float16,fp8,1,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,1,float16,float16,3,0.03721066564321518
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,1,float16,fp8,3,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,1,float16,fp8,31,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,1,float16,float16,7,0.03719999889532725
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,1,float16,fp8,7,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,1,float16,float16,15,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,1,float16,fp8,15,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,1,float16,float16,31,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,1,float16,fp8,31,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,1,float16,float16,63,0.035504000882307686
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,1,float16,fp8,63,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,1,float16,float16,127,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,1,float16,fp8,127,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,1,float16,float16,255,0.038912000755469
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,1,float16,fp8,255,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,1,float16,fp8,511,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,1,float16,float16,511,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,1,float16,float16,1,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,1,float16,fp8,1,0.04744533201058706
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,1,float16,float16,3,0.0576853354771932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,1,float16,fp8,3,0.047450666626294456
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,1,float16,float16,7,0.05972800155480703
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,1,float16,fp8,7,0.04744533201058706
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,1,float16,float16,15,0.058821335434913635
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,1,float16,fp8,15,0.04710933566093445
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,1,float16,float16,31,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,1,float16,fp8,31,0.04744533201058706
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,1,float16,float16,63,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,1,float16,fp8,63,0.04539200166861216
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,1,float16,float16,127,0.060080001751581825
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,1,float16,float16,255,0.011370666325092316
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,1,float16,fp8,127,0.045738667249679565
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,1,float16,float16,255,0.06075733403364817
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,1,float16,fp8,255,0.04778666794300079
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,1,float16,float16,511,0.09830933809280396
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,1,float16,float16,1,0.10478933652242024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,1,float16,fp8,511,0.067930668592453
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,1,float16,fp8,1,0.07816533247629802
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,1,float16,float16,3,0.10478933652242024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,1,float16,fp8,3,0.07679999868075053
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,1,float16,float16,7,0.10411733388900757
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,1,float16,fp8,7,0.07816533247629802
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,1,float16,float16,15,0.10478933652242024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,1,float16,fp8,15,0.07816533247629802
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,1,float16,fp8,31,0.07714133461316426
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,1,float16,float16,31,0.10478933652242024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,1,float16,float16,63,0.10479467113812764
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,1,float16,fp8,63,0.07816533247629802
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,1,float16,float16,127,0.10410666465759277
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,1,float16,fp8,127,0.07986133297284444
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,1,float16,float16,255,0.10513066252072652
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,1,float16,fp8,255,0.07816533247629802
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,1,float16,float16,511,0.17681066195170084
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,1,float16,fp8,511,0.11946666240692139
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,1,float16,float16,1,0.04744533201058706
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,1,float16,fp8,1,0.04539733131726583
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,1,float16,float16,3,0.04744533201058706
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,1,float16,fp8,3,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,1,float16,float16,7,0.047450666626294456
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,1,float16,fp8,7,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,1,float16,float16,15,0.04744533201058706
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,1,float16,fp8,15,0.04369066655635834
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,1,float16,float16,31,0.047450666626294456
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,1,float16,fp8,31,0.04369066655635834
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,1,float16,float16,63,0.04744533201058706
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,1,float16,fp8,63,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,1,float16,float16,127,0.04949333270390829
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,1,float16,fp8,127,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,1,float16,float16,1,0.062122667829195656
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,1,float16,float16,255,0.0481279989083608
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,1,float16,fp8,255,0.04266666869322459
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,1,float16,fp8,1,0.04949333270390829
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,1,float16,float16,3,0.06348800162474315
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,1,float16,fp8,3,0.04949333270390829
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,1,float16,float16,7,0.06382399797439575
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,1,float16,fp8,7,0.049498667319615684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,1,float16,float16,15,0.06382933259010315
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,1,float16,float16,32767,0.039605334401130676
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,1,float16,float16,31,0.0631520003080368
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,1,float16,fp8,15,0.050517335534095764
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,1,float16,fp8,31,0.04949333270390829
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,1,float16,float16,63,0.0634933312733968
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,1,float16,fp8,63,0.04948799808820089
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,1,float16,float16,127,0.06382933259010315
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,1,float16,fp8,127,0.04949333270390829
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,1,float16,float16,255,0.06519466638565063
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,1,float16,float16,1,0.1088853379090627
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,1,float16,fp8,1,0.08295466502507527
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,1,float16,fp8,255,0.052906667192777
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,1,float16,float16,3,0.1088800032933553
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,1,float16,fp8,3,0.08226133386294048
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,1,float16,float16,7,0.10683733224868774
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,1,float16,fp8,7,0.08328533172607422
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,1,float16,float16,15,0.10786666472752889
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,1,float16,fp8,15,0.08226133386294048
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,1,float16,float16,31,0.10752000411351521
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,1,float16,fp8,31,0.08396800359090169
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,1,float16,float16,63,0.10786133011182149
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,1,float16,fp8,63,0.08226133386294048
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,1,float16,fp8,127,0.08363200227419536
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,1,float16,float16,127,0.10683733224868774
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,1,float16,float16,1,0.19490132729212442
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,1,float16,fp8,1,0.145578662554423
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,1,float16,float16,255,0.1088853379090627
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,1,float16,fp8,255,0.08430932958920796
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,1,float16,float16,3,0.19473065932591757
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,1,float16,fp8,3,0.14591999848683676
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,1,float16,fp8,7,0.145578662554423
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,1,float16,float16,7,0.19490132729212442
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,1,float16,float16,15,0.1950719952583313
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,1,float16,fp8,15,0.14575466513633728
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,1,float16,fp8,31,0.14591999848683676
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,1,float16,float16,31,0.19541333119074503
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,1,float16,float16,63,0.19490132729212442
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,1,float16,fp8,63,0.145578662554423
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,1,float16,fp8,127,0.14387200276056925
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,1,float16,float16,127,0.19557867447535196
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,1,float16,float16,1,0.011877333124478659
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,1,float16,fp8,1,0.012351999680201212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,1,float16,float16,255,0.1971199909845988
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,1,float16,float16,3,0.010693332801262537
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,1,float16,fp8,3,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,1,float16,fp8,255,0.14592533310254416
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,1,float16,float16,7,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,1,float16,fp8,7,0.012400000045696894
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,1,float16,float16,15,0.012351999680201212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,1,float16,fp8,15,0.014373333503802618
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,1,float16,float16,31,0.01250133290886879
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,1,float16,fp8,31,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,1,float16,float16,63,0.011418666690587997
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,1,float16,fp8,63,0.013717333475748697
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,1,float16,float16,127,0.012629333883523941
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,1,float16,fp8,127,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,1,float16,float16,255,0.011157333850860596
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,1,float16,fp8,255,0.012298667182525
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,1,float16,float16,511,0.01403733342885971
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,1,float16,fp8,511,0.012576000144084295
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,1,float16,float16,1023,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,1,float16,fp8,1023,0.01331199953953425
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,1,float16,float16,2047,0.018672000616788864
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,1,float16,fp8,2047,0.018709332992633183
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,1,float16,float16,4095,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,1,float16,fp8,4095,0.020768000433842342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,1,float16,float16,8191,0.02218666672706604
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,1,float16,fp8,8191,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,1,float16,float16,16383,0.02526933451493581
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,1,float16,fp8,16383,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,1,float16,float16,32767,0.04266666869322459
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,1,float16,fp8,32767,0.02900800108909607
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,1,float16,float16,65535,0.062122667829195656
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,1,float16,fp8,65535,0.04539733131726583
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,1,float16,float16,1,0.0116799995303154
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,1,float16,fp8,1,0.011253333340088526
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,1,float16,float16,3,0.012319999436537424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,1,float16,fp8,3,0.012629333883523941
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,1,float16,float16,7,0.013888000200192133
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,1,float16,fp8,7,0.011264000087976456
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,1,float16,float16,15,0.013653332988421122
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,1,float16,float16,31,0.012421333541472753
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,1,float16,fp8,31,0.011034666250149408
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,1,float16,float16,63,0.012138667205969492
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,1,float16,fp8,63,0.012181332955757776
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,1,float16,float16,127,0.010687999427318573
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,1,float16,fp8,127,0.01392000044385592
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,1,float16,float16,255,0.01268799975514412
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,1,float16,float16,511,0.012506666282812754
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,1,float16,fp8,511,0.012703999876976013
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,1,float16,float16,1023,0.012319999436537424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,1,float16,fp8,1023,0.012869333227475485
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,1,float16,float16,2047,0.01661866654952367
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,1,float16,fp8,2047,0.016629333297411602
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,1,float16,float16,4095,0.018650667121013004
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,1,float16,fp8,4095,0.01685333376129468
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,1,float16,float16,8191,0.021157334248224895
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,1,float16,fp8,8191,0.01934933289885521
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,1,float16,float16,16383,0.038912000755469
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,1,float16,fp8,16383,0.024911999702453613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,1,float16,float16,32767,0.06007466713587443
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,1,float16,fp8,32767,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,1,float16,float16,65535,0.10103467106819153
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,1,float16,fp8,65535,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,1,float16,float16,1,0.012138667205969492
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,1,float16,fp8,1,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,1,float16,float16,3,0.012346666306257248
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,1,float16,fp8,3,0.012533333152532578
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,1,float16,float16,7,0.012469333906968435
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,1,float16,fp8,7,0.01350933313369751
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,1,float16,float16,15,0.012410666793584824
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,1,float16,fp8,15,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,1,float16,float16,31,0.012074666718641916
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,1,float16,fp8,31,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,1,float16,float16,63,0.012576000144084295
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,1,float16,fp8,63,0.011861333002646765
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,1,float16,float16,127,0.012602667013804117
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,1,float16,fp8,127,0.012202666451533636
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,1,float16,float16,255,0.011605333536863327
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,1,float16,fp8,255,0.012362666428089142
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,1,float16,float16,511,0.013023999830087027
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,1,float16,fp8,511,0.01251199965675672
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,1,float16,float16,1023,0.014570667097965876
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,1,float16,fp8,1023,0.013381333400805792
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,1,float16,float16,2047,0.0186666672428449
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,1,float16,fp8,2047,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,1,float16,float16,4095,0.02218666672706604
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,1,float16,fp8,4095,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,1,float16,fp8,8191,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,1,float16,float16,8191,0.03754133234421412
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,1,float16,float16,16383,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,1,float16,fp8,16383,0.039247999588648476
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,1,float16,float16,32767,0.1013759970664978
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,1,float16,fp8,32767,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,1,float16,float16,65535,0.1829493244489034
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,1,float16,fp8,65535,0.0993280013402303
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,1,float16,float16,1,0.012693333129088083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,1,float16,fp8,1,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,1,float16,float16,3,0.012362666428089142
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,1,float16,fp8,3,0.012351999680201212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,1,float16,float16,7,0.012533333152532578
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,1,float16,fp8,7,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,1,float16,float16,15,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,1,float16,fp8,15,0.013653332988421122
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,1,float16,float16,31,0.012682666381200155
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,1,float16,fp8,31,0.012629333883523941
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,1,float16,float16,63,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,1,float16,fp8,63,0.012858666479587555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,1,float16,float16,127,0.012367999802033106
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,1,float16,fp8,127,0.012506666282812754
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,1,float16,float16,255,0.012346666306257248
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,1,float16,fp8,255,0.012346666306257248
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,1,float16,float16,511,0.013210666676362356
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,1,float16,fp8,511,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,1,float16,float16,1023,0.014592000593741735
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,1,float16,fp8,1023,0.014757333944241205
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,1,float16,float16,2047,0.0210506667693456
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,1,float16,fp8,2047,0.020714666694402695
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,1,float16,float16,4095,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,1,float16,fp8,4095,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,1,float16,float16,8191,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,1,float16,fp8,8191,0.03958933303753535
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,1,float16,float16,16383,0.10205866893132527
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,1,float16,fp8,16383,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,1,float16,float16,32767,0.18653867642084757
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,1,float16,fp8,32767,0.10341866811116536
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,1,float16,float16,65535,0.3548159996668498
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,1,float16,fp8,65535,0.19062399864196777
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,1,float16,float16,1,0.012357333054145178
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,1,float16,fp8,1,0.01097600037852923
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,1,float16,float16,3,0.011488000551859537
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,1,float16,fp8,3,0.010981333752473196
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,1,float16,fp8,7,0.011247999966144562
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,1,float16,float16,15,0.011776000261306763
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,1,float16,fp8,15,0.01081066702802976
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,1,float16,float16,31,0.011328000575304031
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,1,float16,fp8,31,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,1,float16,float16,63,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,1,float16,fp8,63,0.010981333752473196
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,1,float16,float16,127,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,1,float16,fp8,127,0.011770666887362799
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,1,float16,float16,255,0.011328000575304031
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,1,float16,fp8,255,0.011237333218256632
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,1,float16,float16,511,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,1,float16,fp8,511,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,1,float16,float16,1023,0.012858666479587555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,1,float16,fp8,1023,0.013023999830087027
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,1,float16,float16,2047,0.016613333175579708
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,1,float16,fp8,2047,0.016623999923467636
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,1,float16,float16,4095,0.018677332748969395
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,1,float16,fp8,4095,0.01681600014368693
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,1,float16,float16,8191,0.02242133269707362
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,1,float16,fp8,8191,0.020655999581019085
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,1,float16,float16,16383,0.03925866633653641
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,1,float16,fp8,16383,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,1,float16,float16,32767,0.06007466713587443
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,1,float16,fp8,32767,0.04095999896526337
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,1,float16,float16,65535,0.10205866893132527
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,1,float16,fp8,65535,0.06007466713587443
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,1,float16,float16,1,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,1,float16,fp8,1,0.01227733368674914
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,1,float16,float16,3,0.012346666306257248
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,1,float16,fp8,3,0.012666666259368261
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,1,float16,float16,7,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,1,float16,fp8,7,0.011946666985750198
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,1,float16,float16,15,0.01211200033624967
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,1,float16,fp8,15,0.012698666503032049
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,1,float16,float16,31,0.012351999680201212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,1,float16,fp8,31,0.011834666132926941
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,1,float16,float16,63,0.012170666207869848
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,1,float16,fp8,63,0.013007999708255133
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,1,float16,float16,127,0.012133333832025528
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,1,float16,fp8,127,0.012837332983811697
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,1,float16,float16,255,0.014005333185195923
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,1,float16,fp8,255,0.011770666887362799
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,1,float16,float16,511,0.012357333054145178
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,1,float16,fp8,511,0.012847999731699625
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,1,float16,float16,1023,0.014581333845853806
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,1,float16,fp8,1023,0.013194666554530462
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,1,float16,float16,2047,0.018672000616788864
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,1,float16,fp8,2047,0.016965333372354507
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,1,float16,float16,4095,0.022704000274340313
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,1,float16,fp8,4095,0.020714666694402695
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,1,float16,float16,8191,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,1,float16,fp8,8191,0.025258667767047882
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,1,float16,float16,16383,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,1,float16,fp8,16383,0.040618665516376495
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,1,float16,float16,32767,0.1013759970664978
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,1,float16,fp8,32767,0.05973866581916809
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,1,float16,float16,65535,0.18278932571411133
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,1,float16,fp8,65535,0.10000532865524292
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,1,float16,float16,1,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,1,float16,fp8,1,0.012800000607967377
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,1,float16,float16,3,0.012351999680201212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,1,float16,fp8,3,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,1,float16,float16,7,0.013658666362365087
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,1,float16,fp8,7,0.012175999581813812
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,1,float16,float16,15,0.012533333152532578
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,1,float16,fp8,15,0.013679999858140945
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,1,float16,float16,31,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,1,float16,fp8,31,0.01251199965675672
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,1,float16,float16,63,0.01267733300725619
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,1,float16,fp8,63,0.012181332955757776
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,1,float16,float16,127,0.012170666207869848
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,1,float16,fp8,127,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,1,float16,float16,255,0.01268799975514412
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,1,float16,fp8,255,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,1,float16,float16,511,0.01423466702302297
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,1,float16,fp8,511,0.012858666479587555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,1,float16,float16,1023,0.01661866654952367
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,1,float16,fp8,1023,0.014570667097965876
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,1,float16,float16,2047,0.022863999009132385
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,1,float16,fp8,2047,0.02070933332045873
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,1,float16,float16,4095,0.03857066730658213
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,1,float16,fp8,4095,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,1,float16,float16,8191,0.06007466713587443
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,1,float16,fp8,8191,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,1,float16,float16,16383,0.10342400272687276
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,1,float16,fp8,16383,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,1,float16,float16,32767,0.18499734004338583
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,1,float16,fp8,32767,0.10478400190671285
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,1,float16,float16,65535,0.35072000821431476
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,1,float16,fp8,65535,0.18875734011332193
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,1,float16,float16,1,0.011999999483426413
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,1,float16,fp8,1,0.01341333364446958
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,1,float16,float16,3,0.012383999923865
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,1,float16,fp8,3,0.012357333054145178
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,1,float16,float16,7,0.012693333129088083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,1,float16,fp8,7,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,1,float16,float16,15,0.012351999680201212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,1,float16,fp8,15,0.013701333353916803
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,1,float16,float16,31,0.01228800043463707
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,1,float16,fp8,31,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,1,float16,fp8,15,0.013679999858140945
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,1,float16,float16,63,0.012245333443085352
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,1,float16,fp8,63,0.01259200026591619
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,1,float16,float16,127,0.012191999703645706
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,1,float16,fp8,127,0.012421333541472753
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,1,float16,float16,255,0.012986666212479273
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,1,float16,fp8,255,0.011034666250149408
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,1,float16,fp8,255,0.012362666428089142
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,1,float16,float16,511,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,1,float16,fp8,511,0.013568000247081121
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,1,float16,float16,1023,0.013002666334311167
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,1,float16,fp8,1023,0.012693333129088083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,1,float16,float16,2047,0.01239466667175293
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,1,float16,fp8,2047,0.013818666338920593
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,1,float16,float16,4095,0.013482666263977686
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,1,float16,fp8,4095,0.012906666845083237
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,1,float16,float16,8191,0.016656000167131424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,1,float16,fp8,8191,0.019002666076024372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,1,float16,float16,16383,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,1,float16,fp8,16383,0.022805333137512207
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,1,float16,float16,32767,0.03549333413441976
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,1,float16,fp8,32767,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,1,float16,float16,65535,0.03924266745646795
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,1,float16,fp8,65535,0.041296000281969704
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,1,float16,float16,131071,0.043354665239652
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,1,float16,float16,1,0.011786667009194693
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,1,float16,fp8,131071,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,1,float16,fp8,1,0.010693332801262537
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,1,float16,float16,3,0.012367999802033106
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,1,float16,fp8,3,0.011648000528415045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,1,float16,float16,7,0.011781333635250727
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,1,float16,fp8,7,0.011994666109482447
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,1,float16,float16,15,0.010847999403874079
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,1,float16,fp8,15,0.011648000528415045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,1,float16,float16,31,0.012245333443085352
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,1,float16,fp8,31,0.011968000481526056
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,1,float16,float16,63,0.011445333560307821
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,1,float16,fp8,63,0.012789333860079447
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,1,float16,float16,127,0.011802667131026586
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,1,float16,fp8,127,0.01221866657336553
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,1,float16,float16,255,0.01268799975514412
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,1,float16,fp8,255,0.010858666151762009
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,1,float16,float16,511,0.011877333124478659
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,1,float16,fp8,511,0.013493333011865616
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,1,float16,float16,1023,0.011045332998037338
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,1,float16,fp8,1023,0.012223999947309494
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,1,float16,float16,2047,0.014032000054915747
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,1,float16,fp8,2047,0.011898666620254517
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,1,float16,float16,4095,0.012730666746695837
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,1,float16,fp8,4095,0.012858666479587555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,1,float16,float16,8191,0.01672533278663953
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,1,float16,fp8,8191,0.01871466636657715
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,1,float16,float16,16383,0.02492266645034154
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,1,float16,fp8,16383,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,1,float16,float16,32767,0.027301333844661713
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,1,float16,fp8,32767,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,1,float16,float16,65535,0.030037333567937214
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,1,float16,fp8,65535,0.02900800108909607
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,1,float16,float16,1,0.012383999923865
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,1,float16,float16,131071,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,1,float16,fp8,131071,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,1,float16,fp8,1,0.011941333611806234
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,1,float16,float16,3,0.012543999900420507
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,1,float16,fp8,3,0.011871999750534693
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,1,float16,float16,7,0.011616000284751257
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,1,float16,fp8,7,0.012133333832025528
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,1,float16,float16,15,0.011760000139474869
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,1,float16,fp8,15,0.012256000190973282
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,1,float16,float16,31,0.01251199965675672
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,1,float16,fp8,31,0.011071999867757162
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,1,float16,float16,63,0.011402666568756104
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,1,float16,fp8,63,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,1,float16,float16,127,0.012047999848922094
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,1,float16,fp8,127,0.0122079998254776
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,1,float16,float16,255,0.012149333953857422
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,1,float16,fp8,255,0.010863999525705973
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,1,float16,float16,511,0.012181332955757776
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,1,float16,fp8,511,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,1,float16,float16,1023,0.012005332857370377
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,1,float16,fp8,1023,0.013471999516089758
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,1,float16,float16,2047,0.01268799975514412
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,1,float16,fp8,2047,0.011434666812419891
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,1,float16,float16,4095,0.012906666845083237
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,1,float16,fp8,4095,0.013199999928474426
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,1,float16,float16,8191,0.018709332992633183
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,1,float16,fp8,8191,0.018629333625237148
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,1,float16,float16,16383,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,1,float16,fp8,16383,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,1,float16,float16,32767,0.024853333830833435
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,1,float16,fp8,32767,0.023775999744733173
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,1,float16,float16,65535,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,1,float16,fp8,65535,0.024847999215126038
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,1,float16,float16,131071,0.04504533112049103
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,1,float16,fp8,131071,0.03105599929889043
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,1,float16,float16,1,0.011823999385039011
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,1,float16,fp8,1,0.011850666254758835
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,1,float16,float16,3,0.011962667107582092
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,1,float16,fp8,3,0.012991999586423239
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,1,float16,float16,7,0.012794667234023413
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,1,float16,fp8,7,0.011674666156371435
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,1,float16,float16,15,0.011418666690587997
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,1,float16,fp8,15,0.012789333860079447
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,1,float16,float16,31,0.012869333227475485
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,1,float16,fp8,31,0.012346666306257248
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,1,float16,float16,63,0.01227733368674914
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,1,float16,fp8,63,0.012202666451533636
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,1,float16,float16,127,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,1,float16,fp8,127,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,1,float16,float16,255,0.011584000041087469
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,1,float16,fp8,255,0.012256000190973282
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,1,float16,float16,511,0.012693333129088083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,1,float16,fp8,511,0.012624000509579977
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,1,float16,float16,1023,0.013258667041858038
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,1,float16,fp8,1023,0.012869333227475485
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,1,float16,float16,2047,0.013466666142145792
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,1,float16,fp8,2047,0.01444799949725469
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,1,float16,float16,4095,0.013552000125249227
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,1,float16,fp8,4095,0.012906666845083237
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,1,float16,float16,8191,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,1,float16,fp8,8191,0.01899733394384384
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,1,float16,float16,16383,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,1,float16,fp8,16383,0.024847999215126038
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,1,float16,float16,32767,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,1,float16,fp8,32767,0.02867199977238973
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,1,float16,float16,65535,0.029696000119050343
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,1,float16,fp8,65535,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,1,float16,float16,131071,0.036533333361148834
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,1,float16,float16,7,0.012346666306257248
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,1,float16,float16,1,0.013701333353916803
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,1,float16,fp8,131071,0.03311999887228012
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,1,float16,fp8,1,0.012245333443085352
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,1,float16,float16,3,0.012175999581813812
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,1,float16,fp8,3,0.013834666460752487
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,1,float16,float16,7,0.011472000430027643
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,1,float16,fp8,7,0.012256000190973282
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,1,float16,float16,15,0.012629333883523941
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,1,float16,fp8,15,0.012400000045696894
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,1,float16,float16,31,0.012330666184425354
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,1,float16,fp8,31,0.01240533341964086
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,1,float16,float16,63,0.011722666521867117
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,1,float16,fp8,63,0.012794667234023413
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,1,float16,float16,127,0.012042666474978128
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,1,float16,fp8,127,0.012159999459981918
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,1,float16,float16,255,0.012746666868527731
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,1,float16,fp8,255,0.012997332960367203
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,1,float16,float16,511,0.012309333930412928
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,1,float16,fp8,511,0.01219733307758967
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,1,float16,float16,1023,0.013967999567588171
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,1,float16,fp8,1023,0.013717333475748697
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,1,float16,float16,2047,0.013072000195582708
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,1,float16,fp8,2047,0.013823999712864557
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,1,float16,float16,4095,0.014576000471909841
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,1,float16,fp8,4095,0.014394666999578476
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,1,float16,float16,8191,0.01871466636657715
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,1,float16,fp8,8191,0.01871466636657715
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,1,float16,float16,16383,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,1,float16,fp8,16383,0.022757334013779957
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,1,float16,float16,32767,0.024853333830833435
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,1,float16,fp8,32767,0.02383466561635335
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,1,float16,float16,65535,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,1,float16,fp8,65535,0.0266239990790685
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,1,float16,float16,131071,0.04709866642951965
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,1,float16,float16,1,0.012506666282812754
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,1,float16,fp8,131071,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,1,float16,fp8,1,0.012357333054145178
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,1,float16,float16,3,0.01191466674208641
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,1,float16,fp8,3,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,1,float16,float16,7,0.011952000359694162
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,1,float16,fp8,7,0.012624000509579977
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,1,float16,float16,15,0.012794667234023413
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,1,float16,fp8,15,0.012138667205969492
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,1,float16,float16,31,0.011893333246310553
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,1,float16,fp8,31,0.014074667046467463
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,1,float16,float16,63,0.012015999605258306
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,1,float16,fp8,63,0.012351999680201212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,1,float16,float16,127,0.012202666451533636
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,1,float16,fp8,127,0.012645332763592402
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,1,float16,float16,255,0.012410666793584824
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,1,float16,fp8,255,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,1,float16,float16,511,0.012778667112191519
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,1,float16,fp8,511,0.013482666263977686
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,1,float16,float16,1023,0.013327999661366144
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,1,float16,fp8,1023,0.013642666240533194
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,1,float16,float16,2047,0.012794667234023413
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,1,float16,fp8,2047,0.012869333227475485
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,1,float16,float16,4095,0.01672533278663953
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,1,float16,fp8,4095,0.016805333395799
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,1,float16,float16,8191,0.018629333625237148
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,1,float16,fp8,8191,0.019002666076024372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,1,float16,float16,16383,0.02109866589307785
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,1,float16,fp8,16383,0.02070933332045873
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,1,float16,float16,32767,0.024853333830833435
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,1,float16,fp8,32767,0.02276266614596049
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,1,float16,float16,65535,0.04027733455101649
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,1,float16,fp8,65535,0.027647999425729115
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,1,float16,float16,131071,0.06007466713587443
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,1,float16,float16,1,0.011823999385039011
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,1,float16,fp8,131071,0.04197866717974345
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,1,float16,fp8,1,0.012693333129088083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,1,float16,float16,3,0.011952000359694162
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,1,float16,float16,7,0.011317333827416102
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,1,float16,fp8,7,0.01249066616098086
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,1,float16,float16,15,0.012282667060693106
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,1,float16,fp8,15,0.012181332955757776
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,1,float16,float16,31,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,1,float16,fp8,31,0.012400000045696894
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,1,float16,float16,63,0.01314666618903478
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,1,float16,fp8,63,0.010821333775917688
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,1,float16,float16,127,0.011264000087976456
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,1,float16,float16,255,0.01116266722480456
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,1,float16,fp8,127,0.01268799975514412
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,1,float16,float16,511,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,1,float16,fp8,255,0.012554666648308435
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,1,float16,fp8,511,0.01258133351802826
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,1,float16,float16,1023,0.012863999853531519
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,1,float16,fp8,1023,0.012896000097195307
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,1,float16,float16,2047,0.018661333868900936
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,1,float16,fp8,2047,0.018672000616788864
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,1,float16,fp8,4095,0.020714666694402695
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,1,float16,float16,4095,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,1,float16,float16,8191,0.040618665516376495
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,1,float16,fp8,8191,0.02492266645034154
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,1,float16,float16,16383,0.060421332716941833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,1,float16,fp8,16383,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,1,float16,float16,32767,0.10205866893132527
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,1,float16,fp8,32767,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,1,float16,float16,1,0.014362666755914688
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,1,float16,fp8,1,0.012175999581813812
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,1,float16,float16,3,0.011605333536863327
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,1,float16,fp8,3,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,1,float16,float16,7,0.013653332988421122
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,1,float16,fp8,7,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,1,float16,float16,15,0.012965332716703415
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,1,float16,fp8,15,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,1,float16,float16,31,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,1,float16,fp8,31,0.012351999680201212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,1,float16,float16,63,0.01332266628742218
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,1,float16,fp8,63,0.012560000022252401
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,1,float16,float16,127,0.01331199953953425
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,1,float16,fp8,127,0.01331199953953425
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,1,float16,float16,255,0.012741333494583765
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,1,float16,fp8,255,0.012693333129088083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,1,float16,float16,511,0.014384000251690546
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,1,float16,fp8,511,0.012906666845083237
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,1,float16,float16,1023,0.014576000471909841
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,1,float16,fp8,1023,0.015077333897352219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,1,float16,float16,2047,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,1,float16,fp8,2047,0.02070933332045873
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,1,float16,float16,4095,0.038912000755469
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,1,float16,fp8,4095,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,1,float16,float16,8191,0.06144000093142191
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,1,float16,fp8,8191,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,1,float16,float16,16383,0.10102933645248413
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,1,float16,fp8,16383,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,1,float16,float16,32767,0.18500266472498575
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,1,float16,float16,1,0.014730667074521383
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,1,float16,fp8,32767,0.10478933652242024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,1,float16,fp8,1,0.01320533330241839
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,1,float16,float16,3,0.014767999450365702
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,1,float16,fp8,3,0.012730666746695837
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,1,float16,float16,7,0.014576000471909841
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,1,float16,fp8,7,0.01421333352724711
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,1,float16,float16,15,0.014490666488806406
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,1,float16,float16,31,0.014394666999578476
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,1,float16,fp8,15,0.01303999995191892
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,1,float16,fp8,31,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,1,float16,float16,63,0.013194666554530462
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,1,float16,fp8,63,0.014565333724021912
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,1,float16,float16,127,0.014416000495354334
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,1,float16,fp8,127,0.013898666948080063
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,1,float16,float16,255,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,1,float16,fp8,255,0.012730666746695837
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,1,float16,float16,511,0.016602666427691776
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,1,float16,fp8,511,0.014912000546852747
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,1,float16,float16,1023,0.02075733368595441
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,1,float16,fp8,1023,0.018672000616788864
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,1,float16,float16,2047,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,1,float16,fp8,2047,0.02697066714366277
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,1,float16,float16,4095,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,1,float16,fp8,4095,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,1,float16,float16,8191,0.10308266679445903
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,1,float16,fp8,8191,0.06382933259010315
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,1,float16,float16,16383,0.18619734048843384
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,1,float16,fp8,16383,0.10478400190671285
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,1,float16,float16,1,0.01211200033624967
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,1,float16,float16,32767,0.3541333278020223
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,1,float16,fp8,32767,0.19165867567062378
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,1,float16,fp8,1,0.012042666474978128
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,1,float16,float16,3,0.010821333775917688
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,1,float16,fp8,3,0.012117333710193634
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,1,float16,float16,7,0.011989332735538483
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,1,float16,fp8,7,0.012741333494583765
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,1,float16,float16,15,0.012608000387748083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,1,float16,fp8,15,0.011445333560307821
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,1,float16,float16,31,0.011626667032639185
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,1,float16,fp8,31,0.012421333541472753
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,1,float16,float16,63,0.013999999811251959
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,1,float16,fp8,63,0.011893333246310553
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,1,float16,float16,127,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,1,float16,fp8,127,0.012234666695197424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,1,float16,float16,255,0.010992000500361124
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,1,float16,fp8,255,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,1,float16,float16,511,0.011594666788975397
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,1,float16,fp8,511,0.012351999680201212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,1,float16,float16,1023,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,1,float16,float16,2047,0.014576000471909841
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,1,float16,fp8,1023,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,1,float16,fp8,2047,0.014912000546852747
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,1,float16,float16,4095,0.014917333920796713
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,1,float16,fp8,4095,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,1,float16,float16,8191,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,1,float16,fp8,8191,0.021061333517233532
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,1,float16,float16,16383,0.02457600086927414
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,1,float16,fp8,16383,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,1,float16,float16,32767,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,1,float16,fp8,32767,0.024911999702453613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,1,float16,float16,65535,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,1,float16,fp8,65535,0.027989332874615986
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,1,float16,float16,131071,0.04744533201058706
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,1,float16,fp8,131071,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,1,float16,float16,1,0.01231466606259346
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,1,float16,fp8,1,0.012416000167528788
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,1,float16,float16,3,0.011722666521867117
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,1,float16,fp8,3,0.011434666812419891
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,1,float16,float16,7,0.011034666250149408
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,1,float16,float16,15,0.011407999942700068
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,1,float16,fp8,15,0.010863999525705973
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,1,float16,float16,31,0.011946666985750198
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,1,float16,fp8,31,0.010933333386977514
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,1,float16,float16,63,0.010981333752473196
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,1,float16,fp8,63,0.010853332777818045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,1,float16,float16,127,0.01209066684047381
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,1,float16,fp8,127,0.011029332876205444
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,1,float16,float16,255,0.011952000359694162
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,1,float16,fp8,255,0.010687999427318573
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,1,float16,float16,511,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,1,float16,fp8,511,0.010858666151762009
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,1,float16,fp8,1023,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,1,float16,float16,1023,0.012362666428089142
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,1,float16,float16,2047,0.0129120002190272
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,1,float16,fp8,2047,0.014544000228246054
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,1,float16,float16,4095,0.01870399961868922
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,1,float16,fp8,4095,0.018672000616788864
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,1,float16,float16,8191,0.02081599955757459
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,1,float16,fp8,8191,0.020714666694402695
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,1,float16,float16,16383,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,1,float16,fp8,16383,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,1,float16,float16,32767,0.025253333151340485
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,1,float16,fp8,32767,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,1,float16,float16,65535,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,1,float16,fp8,65535,0.028688001135985058
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,1,float16,float16,131071,0.06417066852251689
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,1,float16,fp8,131071,0.04505600035190582
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,1,float16,float16,1,0.010351999973257383
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,1,float16,fp8,1,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,1,float16,float16,3,0.011264000087976456
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,1,float16,fp8,3,0.012351999680201212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,1,float16,float16,7,0.010757333288590113
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,1,float16,fp8,7,0.012351999680201212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,1,float16,float16,15,0.010351999973257383
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,1,float16,fp8,15,0.012693333129088083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,1,float16,float16,31,0.012026666353146235
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,1,float16,fp8,31,0.012693333129088083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,1,float16,float16,63,0.011087999989589056
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,1,float16,fp8,63,0.012128000458081564
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,1,float16,float16,127,0.01091733326514562
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,1,float16,fp8,127,0.011258666714032492
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,1,float16,float16,255,0.011029332876205444
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,1,float16,fp8,255,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,1,float16,float16,511,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,1,float16,fp8,511,0.012357333054145178
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,1,float16,float16,1023,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,1,float16,fp8,1023,0.013317332913478216
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,1,float16,float16,2047,0.016666666915019352
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,1,float16,fp8,2047,0.01700266698996226
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,1,float16,float16,4095,0.01870399961868922
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,1,float16,fp8,4095,0.019007999449968338
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,1,float16,float16,8191,0.02075733368595441
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,1,float16,fp8,8191,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,1,float16,float16,16383,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,1,float16,fp8,16383,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,1,float16,fp8,32767,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,1,float16,float16,32767,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,1,float16,float16,65535,0.060421332716941833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,1,float16,fp8,65535,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,1,float16,float16,131071,0.10103467106819153
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,1,float16,fp8,131071,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,1,float16,float16,1,0.01414399966597557
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,1,float16,fp8,1,0.012389333297808966
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,1,float16,float16,3,0.012005332857370377
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,1,float16,fp8,3,0.012629333883523941
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,1,float16,float16,7,0.012538666526476542
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,1,float16,fp8,7,0.012560000022252401
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,1,float16,float16,15,0.012533333152532578
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,1,float16,fp8,15,0.012335999558369318
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,1,float16,float16,31,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,1,float16,fp8,31,0.010847999403874079
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,1,float16,float16,63,0.012341332932313284
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,1,float16,fp8,63,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,1,float16,float16,127,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,1,float16,fp8,127,0.011018666128317514
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,1,float16,float16,255,0.012351999680201212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,1,float16,fp8,255,0.012400000045696894
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,1,float16,float16,511,0.013034666577974955
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,1,float16,fp8,511,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,1,float16,float16,1023,0.014671999961137772
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,1,float16,fp8,1023,0.014778666198253632
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,1,float16,float16,2047,0.02275199939807256
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,1,float16,fp8,2047,0.020746666938066483
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,1,float16,float16,4095,0.039936001102129616
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,1,float16,fp8,4095,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,1,float16,float16,8191,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,1,float16,fp8,8191,0.03924266745646795
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,1,float16,float16,16383,0.10308266679445903
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,1,float16,fp8,16383,0.06007466713587443
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,1,float16,float16,1,0.014570667097965876
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,1,float16,fp8,1,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,1,float16,float16,3,0.014677333335081736
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,1,float16,fp8,3,0.012730666746695837
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,1,float16,float16,7,0.014570667097965876
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,1,float16,fp8,7,0.012560000022252401
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,1,float16,float16,15,0.01458666721979777
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,1,float16,fp8,15,0.012736000120639801
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,1,float16,float16,31,0.013194666554530462
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,1,float16,fp8,31,0.012906666845083237
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,1,float16,float16,63,0.014394666999578476
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,1,float16,fp8,63,0.012730666746695837
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,1,float16,float16,127,0.014757333944241205
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,1,float16,fp8,127,0.012885333349307379
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,1,float16,float16,255,0.013893333574136099
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,1,float16,fp8,255,0.012906666845083237
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,1,float16,float16,511,0.016623999923467636
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,1,float16,fp8,511,0.014602666099866232
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,1,float16,float16,1023,0.02070933332045873
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,1,float16,fp8,1023,0.016666666915019352
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,1,float16,float16,2047,0.04027733455101649
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,1,float16,fp8,2047,0.02697066714366277
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,1,float16,float16,4095,0.062122667829195656
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,1,float16,fp8,4095,0.041296000281969704
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,1,float16,float16,8191,0.10409599542617798
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,1,float16,fp8,8191,0.062458669145902
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,1,float16,float16,16383,0.18756266434987387
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,1,float16,fp8,16383,0.10683733224868774
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,1,float16,float16,1,0.016623999923467636
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,1,float16,fp8,1,0.014677333335081736
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,1,float16,float16,3,0.016810666769742966
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,1,float16,fp8,3,0.014773332824309668
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,1,float16,float16,7,0.016965333372354507
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,1,float16,fp8,7,0.014762666076421738
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,1,float16,float16,15,0.016623999923467636
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,1,float16,fp8,15,0.014943999548753103
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,1,float16,float16,31,0.016965333372354507
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,1,float16,fp8,31,0.014767999450365702
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,1,float16,float16,63,0.01684800038735072
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,1,float16,fp8,63,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,1,float16,float16,127,0.016949333250522614
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,1,float16,fp8,127,0.014677333335081736
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,1,float16,float16,255,0.016970666746298473
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,1,float16,fp8,255,0.014869333555301031
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,1,float16,float16,511,0.021162666380405426
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,1,float16,fp8,511,0.018709332992633183
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,1,float16,float16,1023,0.03583999971548716
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,1,float16,fp8,1023,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,1,float16,float16,2047,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,1,float16,fp8,2047,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,1,float16,float16,4095,0.10478933652242024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,1,float16,fp8,4095,0.06417066852251689
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,1,float16,float16,8191,0.1884160041809082
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,1,float16,fp8,8191,0.10684266686439514
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,1,float16,float16,16383,0.35653332869211835
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,1,float16,fp8,16383,0.19267733891805014
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,1,float16,float16,1,0.014576000471909841
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,1,float16,fp8,1,0.012917333592971167
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,1,float16,float16,3,0.014565333724021912
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,1,float16,fp8,3,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,1,float16,float16,7,0.01458666721979777
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,1,float16,fp8,7,0.012741333494583765
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,1,float16,float16,15,0.014757333944241205
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,1,float16,fp8,15,0.012896000097195307
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,1,float16,float16,31,0.014581333845853806
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,1,float16,fp8,31,0.012901333471139273
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,1,float16,float16,63,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,1,float16,fp8,63,0.012906666845083237
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,1,float16,float16,127,0.01441066712141037
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,1,float16,fp8,127,0.0129120002190272
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,1,float16,float16,255,0.014570667097965876
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,1,float16,fp8,255,0.012906666845083237
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,1,float16,float16,511,0.016810666769742966
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,1,float16,fp8,511,0.01461333284775416
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,1,float16,float16,1023,0.020714666694402695
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,1,float16,fp8,1023,0.01870399961868922
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,1,float16,fp8,3,0.01166933278242747
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,1,float16,float16,2047,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,1,float16,fp8,2047,0.02696000039577484
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,1,float16,float16,4095,0.0628053347269694
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,1,float16,fp8,4095,0.041984001795450844
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,1,float16,float16,1,0.01672533278663953
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,1,float16,fp8,1,0.014959999670584997
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,1,float16,float16,3,0.016672000288963318
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,1,float16,float16,7,0.01666133354107539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,1,float16,fp8,7,0.014848000059525171
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,1,float16,float16,15,0.016794666647911072
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,1,float16,fp8,15,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,1,float16,float16,31,0.016858667135238647
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,1,float16,fp8,31,0.015029333531856537
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,1,float16,float16,63,0.01655999943614006
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,1,float16,fp8,63,0.014848000059525171
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,1,float16,float16,127,0.016895999511082966
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,1,float16,fp8,127,0.014746667196353277
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,1,float16,float16,255,0.016832000265518825
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,1,float16,fp8,255,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,1,float16,float16,511,0.020981334149837494
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,1,float16,fp8,511,0.018698666244745255
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,1,float16,float16,1023,0.03549866626660029
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,1,float16,fp8,1023,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,1,float16,float16,2047,0.0628053347269694
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,1,float16,fp8,2047,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,1,float16,float16,4095,0.10479467113812764
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,1,float16,fp8,4095,0.06383466720581055
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,1,float16,float16,1,0.021781332790851593
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,1,float16,fp8,1,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,1,float16,float16,3,0.022800001005331676
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,1,float16,fp8,3,0.018709332992633183
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,1,float16,float16,7,0.022800001005331676
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,1,float16,fp8,7,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,1,float16,float16,15,0.022800001005331676
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,1,float16,fp8,15,0.01870399961868922
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,1,float16,float16,31,0.022800001005331676
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,1,float16,fp8,31,0.01871466636657715
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,1,float16,float16,63,0.022874665757020313
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,1,float16,fp8,63,0.018709332992633183
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,1,float16,float16,127,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,1,float16,fp8,127,0.018698666244745255
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,1,float16,float16,255,0.022800001005331676
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,1,float16,fp8,255,0.01870399961868922
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,1,float16,float16,511,0.034474665919939675
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,1,float16,fp8,511,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,1,float16,float16,1023,0.05494933327039083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,1,float16,fp8,1023,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,1,float16,float16,2047,0.09693866968154907
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,1,float16,fp8,2047,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,1,float16,float16,4095,0.17493865887324014
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,1,float16,fp8,4095,0.09830400347709656
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,1,float16,float16,1,0.01700266698996226
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,1,float16,fp8,1,0.016672000288963318
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,1,float16,float16,3,0.01666133354107539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,1,float16,fp8,3,0.01666133354107539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,1,float16,float16,7,0.016805333395799
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,1,float16,fp8,7,0.01666133354107539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,1,float16,float16,15,0.01681600014368693
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,1,float16,fp8,15,0.016645333419243496
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,1,float16,float16,31,0.016666666915019352
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,1,float16,fp8,31,0.016666666915019352
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,1,float16,float16,63,0.016805333395799
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,1,float16,fp8,63,0.016549333930015564
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,1,float16,float16,127,0.016810666769742966
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,1,float16,fp8,127,0.014607999473810196
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,1,float16,float16,255,0.016890666137139004
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,1,float16,fp8,255,0.014607999473810196
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,1,float16,float16,511,0.022815999885400135
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,1,float16,fp8,511,0.01870399961868922
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,1,float16,float16,1023,0.03583466758330663
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,1,float16,fp8,1023,0.022858666876951855
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,1,float16,float16,2047,0.06382933259010315
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,1,float16,fp8,2047,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,1,float16,float16,1,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,1,float16,fp8,1,0.019039999693632126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,1,float16,float16,3,0.022805333137512207
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,1,float16,fp8,3,0.01878400022784869
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,1,float16,float16,7,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,1,float16,fp8,7,0.01912533367673556
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,1,float16,float16,15,0.022800001005331676
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,1,float16,fp8,15,0.020474666108687718
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,1,float16,fp8,7,0.011434666812419891
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,1,float16,float16,31,0.022805333137512207
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,1,float16,fp8,31,0.020143999407688778
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,1,float16,float16,63,0.022863999009132385
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,1,float16,fp8,63,0.018709332992633183
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,1,float16,float16,127,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,1,float16,fp8,127,0.020714666694402695
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,1,float16,float16,255,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,1,float16,fp8,255,0.018687999496857326
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,1,float16,float16,511,0.03446933378775915
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,1,float16,fp8,511,0.026288000245889027
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,1,float16,float16,1023,0.05563733478387197
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,1,float16,fp8,1023,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,1,float16,float16,1,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,1,float16,float16,2047,0.09967466195424397
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,1,float16,fp8,2047,0.05974400043487549
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,1,float16,fp8,1,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,1,float16,float16,3,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,1,float16,fp8,3,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,1,float16,float16,7,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,1,float16,fp8,7,0.02697066714366277
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,1,float16,float16,15,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,1,float16,fp8,15,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,1,float16,float16,31,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,1,float16,fp8,31,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,1,float16,float16,63,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,1,float16,fp8,63,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,1,float16,float16,127,0.03309866786003113
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,1,float16,fp8,127,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,1,float16,float16,255,0.03583999971548716
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,1,float16,fp8,255,0.027306665976842243
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,1,float16,float16,511,0.05563733478387197
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,1,float16,fp8,511,0.04028266668319702
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,1,float16,float16,1023,0.09454933802286784
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,1,float16,fp8,1023,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,1,float16,float16,2047,0.17559999227523804
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,1,float16,fp8,2047,0.10069333513577779
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,1,float16,float16,1,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,1,float16,fp8,1,0.013781332721312841
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,1,float16,float16,3,0.01191466674208641
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,1,float16,fp8,3,0.012693333129088083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,1,float16,float16,7,0.012026666353146235
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,1,float16,fp8,7,0.011999999483426413
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,1,float16,float16,15,0.012117333710193634
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,1,float16,fp8,15,0.01331199953953425
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,1,float16,float16,31,0.01192533348997434
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,1,float16,fp8,31,0.013045333325862885
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,1,float16,float16,63,0.012666666259368261
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,1,float16,fp8,63,0.012549333274364471
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,1,float16,float16,127,0.012133333832025528
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,1,float16,fp8,127,0.012800000607967377
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,1,float16,float16,255,0.012069333344697952
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,1,float16,fp8,255,0.012421333541472753
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,1,float16,float16,511,0.01357866699496905
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,1,float16,fp8,511,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,1,float16,float16,1023,0.012938667088747025
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,1,float16,fp8,1023,0.013653332988421122
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,1,float16,float16,2047,0.014597332725922266
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,1,float16,fp8,2047,0.01458666721979777
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,1,float16,float16,4095,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,1,float16,fp8,4095,0.01871466636657715
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,1,float16,float16,8191,0.021407999098300934
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,1,float16,fp8,8191,0.02065066620707512
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,1,float16,float16,16383,0.022885332504908245
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,1,float16,fp8,16383,0.02275199939807256
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,1,float16,float16,32767,0.025258667767047882
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,1,float16,fp8,32767,0.02279466638962428
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,1,float16,float16,65535,0.04265599946180979
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,1,float16,fp8,65535,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,1,float16,float16,131071,0.06417066852251689
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,1,float16,float16,1,0.012304000556468964
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,1,float16,fp8,131071,0.044719999035199486
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,1,float16,fp8,1,0.01228800043463707
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,1,float16,float16,3,0.012960000584522883
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,1,float16,fp8,3,0.012586666891972223
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,1,float16,float16,7,0.01163200040658315
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,1,float16,fp8,7,0.013744000345468521
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,1,float16,float16,15,0.011711999773979187
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,1,float16,fp8,15,0.01190399999419848
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,1,float16,float16,31,0.012261333564917246
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,1,float16,fp8,31,0.01219733307758967
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,1,float16,float16,63,0.01458666721979777
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,1,float16,fp8,63,0.012901333471139273
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,1,float16,float16,127,0.012234666695197424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,1,float16,fp8,127,0.012634667257467905
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,1,float16,float16,255,0.012432000289360682
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,1,float16,fp8,255,0.012229333321253458
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,1,float16,float16,511,0.013418667018413544
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,1,float16,fp8,511,0.01331199953953425
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,1,float16,float16,1023,0.012693333129088083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,1,float16,fp8,1023,0.012543999900420507
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,1,float16,float16,2047,0.01666133354107539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,1,float16,fp8,2047,0.017290666699409485
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,1,float16,float16,4095,0.01869333287080129
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,1,float16,fp8,4095,0.018709332992633183
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,1,float16,float16,8191,0.021157334248224895
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,1,float16,fp8,8191,0.01904533306757609
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,1,float16,float16,16383,0.024853333830833435
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,1,float16,fp8,16383,0.022757334013779957
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,1,float16,float16,32767,0.039247999588648476
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,1,float16,fp8,32767,0.02696000039577484
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,1,float16,float16,65535,0.06144000093142191
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,1,float16,fp8,65535,0.039594667653242745
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,1,float16,float16,131071,0.10274133086204529
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,1,float16,fp8,131071,0.062463998794555664
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,1,float16,float16,1,0.012117333710193634
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,1,float16,fp8,1,0.01313599944114685
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,1,float16,float16,3,0.012432000289360682
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,1,float16,fp8,3,0.012389333297808966
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,1,float16,float16,7,0.012400000045696894
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,1,float16,fp8,7,0.011871999750534693
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,1,float16,float16,15,0.012389333297808966
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,1,float16,fp8,15,0.013338666409254074
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,1,float16,float16,31,0.012453333785136541
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,1,float16,fp8,31,0.012074666718641916
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,1,float16,float16,63,0.012346666306257248
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,1,float16,fp8,63,0.012432000289360682
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,1,float16,float16,127,0.011898666620254517
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,1,float16,fp8,127,0.013034666577974955
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,1,float16,float16,255,0.012170666207869848
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,1,float16,fp8,255,0.012400000045696894
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,1,float16,float16,511,0.012362666428089142
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,1,float16,fp8,511,0.012698666503032049
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,1,float16,float16,1023,0.012458667159080505
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,1,float16,fp8,1023,0.01358933374285698
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,1,float16,float16,2047,0.016634666671355564
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,1,float16,fp8,2047,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,1,float16,float16,4095,0.01859733338157336
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,1,float16,fp8,4095,0.016607999801635742
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,1,float16,float16,8191,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,1,float16,fp8,8191,0.01894933357834816
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,1,float16,float16,16383,0.03754666695992152
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,1,float16,fp8,16383,0.02492266645034154
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,1,float16,float16,32767,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,1,float16,fp8,32767,0.038912000755469
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,1,float16,float16,65535,0.10034666458765666
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,1,float16,fp8,65535,0.05905066430568695
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,1,float16,float16,131071,0.1838080088297526
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,1,float16,fp8,131071,0.1013759970664978
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,1,float16,float16,1,0.022874665757020313
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,1,float16,fp8,1,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,1,float16,float16,3,0.02314666658639908
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,1,float16,fp8,3,0.02075733368595441
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,1,float16,float16,7,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,1,float16,fp8,7,0.02075200031201045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,1,float16,float16,15,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,1,float16,fp8,15,0.020714666694402695
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,1,float16,float16,31,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,1,float16,fp8,31,0.02075733368595441
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,1,float16,float16,63,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,1,float16,fp8,63,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,1,float16,float16,127,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,1,float16,fp8,127,0.02082666630546252
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,1,float16,float16,255,0.023141334454218548
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,1,float16,fp8,255,0.01904533306757609
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,1,float16,float16,511,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,1,float16,fp8,511,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,1,float16,float16,1023,0.05563733478387197
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,1,float16,fp8,1023,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,1,float16,float16,1,0.03309866786003113
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,1,float16,fp8,1,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,1,float16,float16,3,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,1,float16,fp8,3,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,1,float16,float16,7,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,1,float16,fp8,7,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,1,float16,float16,15,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,1,float16,fp8,15,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,1,float16,float16,31,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,1,float16,fp8,31,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,1,float16,float16,63,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,1,float16,fp8,63,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,1,float16,float16,127,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,1,float16,fp8,127,0.02867199977238973
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,1,float16,float16,255,0.03583466758330663
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,1,float16,fp8,255,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,1,float16,float16,511,0.05563733478387197
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,1,float16,fp8,511,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,1,float16,float16,1,0.05701333284378052
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,1,float16,float16,1023,0.09660800298055013
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,1,float16,fp8,1023,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,1,float16,fp8,1,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,1,float16,float16,3,0.0576853354771932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,1,float16,fp8,3,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,1,float16,float16,7,0.0576853354771932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,1,float16,fp8,7,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,1,float16,float16,15,0.05563733478387197
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,1,float16,fp8,15,0.043354665239652
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,1,float16,float16,31,0.05598400036493937
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,1,float16,fp8,31,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,1,float16,float16,63,0.0576853354771932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,1,float16,fp8,63,0.04334400097529093
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,1,float16,float16,127,0.0576853354771932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,1,float16,fp8,127,0.043354665239652
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,1,float16,float16,255,0.05904533465703329
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,1,float16,fp8,255,0.04539733131726583
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,1,float16,float16,511,0.09489066402117412
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,1,float16,fp8,511,0.06587733328342438
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,1,float16,float16,1023,0.17151999473571777
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,1,float16,float16,1,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,1,float16,fp8,1023,0.10274133086204529
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,1,float16,fp8,1,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,1,float16,float16,3,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,1,float16,fp8,3,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,1,float16,float16,7,0.036864000062147774
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,1,float16,fp8,7,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,1,float16,float16,15,0.036858665446440377
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,1,float16,fp8,15,0.031066666046778362
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,1,float16,float16,31,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,1,float16,fp8,31,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,1,float16,float16,63,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,1,float16,fp8,63,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,1,float16,float16,127,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,1,float16,fp8,127,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,1,float16,fp8,3,0.016613333175579708
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,1,float16,float16,255,0.03925866633653641
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,1,float16,fp8,255,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,1,float16,float16,511,0.0576853354771932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,1,float16,float16,1,0.0576853354771932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,1,float16,fp8,1,0.04540266593297323
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,1,float16,fp8,511,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,1,float16,float16,3,0.05973866581916809
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,1,float16,fp8,3,0.04539733131726583
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,1,float16,float16,7,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,1,float16,fp8,7,0.04540266593297323
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,1,float16,float16,15,0.059392000238100685
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,1,float16,fp8,15,0.04539733131726583
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,1,float16,float16,31,0.05836800237496694
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,1,float16,fp8,31,0.04540266593297323
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,1,float16,float16,63,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,1,float16,fp8,63,0.04539733131726583
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,1,float16,float16,127,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,1,float16,fp8,127,0.04539733131726583
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,1,float16,float16,255,0.060415998101234436
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,1,float16,fp8,255,0.04744533201058706
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,1,float16,float16,511,0.09830933809280396
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,1,float16,float16,1,0.10342400272687276
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,1,float16,fp8,1,0.07816533247629802
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,1,float16,fp8,511,0.0679253339767456
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,1,float16,float16,3,0.10478933652242024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,1,float16,fp8,3,0.07918933530648549
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,1,float16,float16,7,0.10478400190671285
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,1,float16,fp8,7,0.07816533247629802
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,1,float16,float16,15,0.1037600040435791
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,1,float16,fp8,15,0.07817066709200542
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,1,float16,float16,31,0.10478400190671285
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,1,float16,fp8,31,0.07816533247629802
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,1,float16,float16,63,0.10308266679445903
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,1,float16,fp8,63,0.07850666840871175
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,1,float16,float16,127,0.10479467113812764
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,1,float16,fp8,127,0.07850666840871175
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,1,float16,float16,255,0.1058079997698466
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,1,float16,fp8,255,0.08021333316961925
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,1,float16,float16,1,0.01268799975514412
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,1,float16,fp8,1,0.014069333672523499
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,1,float16,float16,511,0.1752799948056539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,1,float16,fp8,511,0.11980799833933513
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,1,float16,float16,3,0.01210133358836174
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,1,float16,fp8,3,0.013584000368913015
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,1,float16,float16,7,0.012213333199421564
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,1,float16,fp8,7,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,1,float16,fp8,15,0.01239466667175293
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,1,float16,float16,31,0.012186666329701742
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,1,float16,fp8,31,0.012357333054145178
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,1,float16,float16,63,0.012469333906968435
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,1,float16,fp8,63,0.012576000144084295
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,1,float16,float16,127,0.012159999459981918
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,1,float16,fp8,127,0.012426666915416718
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,1,float16,float16,255,0.010687999427318573
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,1,float16,fp8,255,0.012351999680201212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,1,float16,float16,511,0.012400000045696894
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,1,float16,fp8,511,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,1,float16,float16,1023,0.012693333129088083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,1,float16,fp8,1023,0.012970666090647379
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,1,float16,float16,2047,0.016879999389251072
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,1,float16,fp8,2047,0.016965333372354507
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,1,float16,float16,4095,0.01860800012946129
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,1,float16,fp8,4095,0.018719999740521114
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,1,float16,float16,8191,0.02083733429511388
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,1,float16,fp8,8191,0.02075733368595441
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,1,float16,float16,16383,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,1,float16,fp8,16383,0.02276800076166789
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,1,float16,float16,32767,0.039247999588648476
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,1,float16,fp8,32767,0.02697066714366277
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,1,float16,float16,65535,0.06177600224812826
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,1,float16,fp8,65535,0.04062400013208389
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,1,float16,float16,131071,0.10274133086204529
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,1,float16,float16,1,0.012229333321253458
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,1,float16,fp8,131071,0.06211199859778086
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,1,float16,fp8,1,0.01109333336353302
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,1,float16,float16,3,0.0124746672809124
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,1,float16,fp8,3,0.013141332815090815
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,1,float16,float16,7,0.011973333855470022
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,1,float16,fp8,7,0.012608000387748083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,1,float16,float16,15,0.011157333850860596
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,1,float16,fp8,15,0.011877333124478659
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,1,float16,float16,31,0.012426666915416718
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,1,float16,fp8,31,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,1,float16,float16,63,0.012159999459981918
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,1,float16,fp8,63,0.012389333297808966
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,1,float16,float16,127,0.012847999731699625
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,1,float16,fp8,127,0.01198400060335795
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,1,float16,float16,255,0.012645332763592402
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,1,float16,fp8,255,0.012362666428089142
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,1,float16,float16,511,0.014165333161751429
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,1,float16,fp8,511,0.012970666090647379
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,1,float16,float16,1023,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,1,float16,fp8,1023,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,1,float16,float16,2047,0.016666666915019352
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,1,float16,fp8,2047,0.01661866654952367
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,1,float16,float16,4095,0.018698666244745255
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,1,float16,fp8,4095,0.016778666526079178
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,1,float16,float16,8191,0.021045332153638203
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,1,float16,fp8,8191,0.020762667059898376
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,1,float16,float16,16383,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,1,float16,fp8,16383,0.02492800106604894
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,1,float16,float16,32767,0.060080001751581825
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,1,float16,fp8,32767,0.039594667653242745
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,1,float16,float16,65535,0.10069333513577779
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,1,float16,fp8,65535,0.05905066430568695
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,1,float16,float16,131071,0.18516800800959268
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,1,float16,fp8,131071,0.10240000486373901
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,1,float16,float16,1,0.01173866664369901
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,1,float16,fp8,1,0.013007999708255133
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,1,float16,float16,3,0.012357333054145178
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,1,float16,fp8,3,0.011770666887362799
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,1,float16,float16,7,0.012538666526476542
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,1,float16,fp8,7,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,1,float16,float16,15,0.010682666053374609
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,1,float16,fp8,15,0.012938667088747025
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,1,float16,float16,31,0.012458667159080505
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,1,float16,fp8,31,0.011205332974592844
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,1,float16,float16,63,0.01258133351802826
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,1,float16,fp8,63,0.011962667107582092
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,1,float16,float16,127,0.012106666962305704
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,1,float16,fp8,127,0.012319999436537424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,1,float16,float16,255,0.014416000495354334
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,1,float16,fp8,255,0.013327999661366144
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,1,float16,float16,511,0.013605333864688873
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,1,float16,fp8,511,0.013034666577974955
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,1,float16,float16,1023,0.014335999886194864
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,1,float16,fp8,1023,0.014282666146755219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,1,float16,float16,2047,0.017680000513792038
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,1,float16,fp8,2047,0.016917333006858826
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,1,float16,float16,4095,0.02089600016673406
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,1,float16,fp8,4095,0.02070933332045873
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,1,float16,float16,8191,0.03721066564321518
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,1,float16,fp8,8191,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,1,float16,float16,16383,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,1,float16,fp8,16383,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,1,float16,float16,32767,0.09966933727264404
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,1,float16,fp8,32767,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,1,float16,float16,65535,0.18210132916768393
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,1,float16,fp8,65535,0.10069866975148518
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,1,float16,float16,131071,0.34867199261983234
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,1,float16,fp8,131071,0.18107734123865762
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,1,float16,float16,1,0.011898666620254517
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,1,float16,fp8,1,0.012362666428089142
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,1,float16,float16,3,0.012229333321253458
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,1,float16,fp8,3,0.012378666549921036
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,1,float16,float16,7,0.013141332815090815
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,1,float16,fp8,7,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,1,float16,float16,15,0.012879999975363413
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,1,float16,fp8,15,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,1,float16,float16,31,0.012149333953857422
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,1,float16,fp8,31,0.012229333321253458
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,1,float16,float16,63,0.010992000500361124
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,1,float16,fp8,63,0.012960000584522883
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,1,float16,float16,127,0.012165332833925882
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,1,float16,fp8,127,0.01313599944114685
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,1,float16,float16,255,0.01219733307758967
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,1,float16,fp8,255,0.012495999534924826
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,1,float16,float16,511,0.012357333054145178
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,1,float16,fp8,511,0.012389333297808966
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,1,float16,float16,1023,0.012970666090647379
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,1,float16,fp8,1023,0.01303999995191892
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,1,float16,float16,2047,0.016672000288963318
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,1,float16,fp8,2047,0.01672533278663953
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,1,float16,float16,4095,0.018672000616788864
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,1,float16,fp8,4095,0.016805333395799
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,1,float16,float16,8191,0.02178666740655899
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,1,float16,fp8,8191,0.020714666694402695
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,1,float16,float16,16383,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,1,float16,fp8,16383,0.025546667476495106
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,1,float16,float16,32767,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,1,float16,fp8,32767,0.03925866633653641
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,1,float16,float16,65535,0.10240000486373901
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,1,float16,fp8,65535,0.060415998101234436
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,1,float16,float16,131071,0.1868799924850464
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,1,float16,float16,1,0.011328000575304031
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,1,float16,fp8,131071,0.10274133086204529
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,1,float16,fp8,1,0.011434666812419891
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,1,float16,float16,3,0.010847999403874079
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,1,float16,fp8,3,0.01246400053302447
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,1,float16,float16,7,0.012826666235923767
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,1,float16,fp8,7,0.010687999427318573
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,1,float16,float16,15,0.012074666718641916
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,1,float16,fp8,15,0.01268799975514412
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,1,float16,float16,31,0.011946666985750198
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,1,float16,fp8,31,0.013909333695967993
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,1,float16,fp8,63,0.011413333316644033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,1,float16,float16,63,0.011855999628702799
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,1,float16,float16,127,0.01240533341964086
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,1,float16,fp8,127,0.012170666207869848
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,1,float16,float16,255,0.012346666306257248
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,1,float16,fp8,255,0.011920000116030375
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,1,float16,float16,511,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,1,float16,fp8,511,0.012863999853531519
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,1,float16,float16,1023,0.01461333284775416
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,1,float16,fp8,1023,0.012736000120639801
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,1,float16,float16,2047,0.0186666672428449
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,1,float16,fp8,2047,0.016805333395799
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,1,float16,float16,4095,0.02075200031201045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,1,float16,fp8,4095,0.02072000006834666
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,1,float16,float16,8191,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,1,float16,fp8,8191,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,1,float16,float16,16383,0.06007466713587443
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,1,float16,fp8,16383,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,1,float16,float16,32767,0.10103467106819153
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,1,float16,fp8,32767,0.05972800155480703
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,1,float16,float16,65535,0.1838080088297526
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,1,float16,fp8,65535,0.09965866804122925
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,1,float16,float16,131071,0.34798399607340497
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,1,float16,fp8,131071,0.18244266510009766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,1,float16,float16,1,0.011434666812419891
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,1,float16,fp8,1,0.011893333246310553
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,1,float16,float16,3,0.010682666053374609
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,1,float16,fp8,3,0.011952000359694162
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,1,float16,float16,7,0.01192533348997434
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,1,float16,fp8,7,0.011087999989589056
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,1,float16,float16,15,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,1,float16,fp8,15,0.011941333611806234
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,1,float16,float16,31,0.010853332777818045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,1,float16,fp8,31,0.012096000214417776
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,1,float16,float16,63,0.011706666400035223
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,1,float16,fp8,63,0.010842667271693548
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,1,float16,float16,127,0.011450666934251785
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,1,float16,fp8,127,0.013487999637921652
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,1,float16,float16,255,0.011242666592200598
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,1,float16,fp8,255,0.012346666306257248
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,1,float16,float16,511,0.013562666873137156
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,1,float16,fp8,511,0.012047999848922094
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,1,float16,float16,1023,0.012186666329701742
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,1,float16,fp8,1023,0.013647999614477158
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,1,float16,float16,2047,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,1,float16,fp8,2047,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,1,float16,float16,4095,0.01303999995191892
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,1,float16,fp8,4095,0.012576000144084295
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,1,float16,float16,8191,0.016789333273967106
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,1,float16,fp8,8191,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,1,float16,float16,16383,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,1,float16,fp8,16383,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,1,float16,float16,32767,0.03310399999221166
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,1,float16,fp8,32767,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,1,float16,float16,65535,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,1,float16,fp8,65535,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,1,float16,float16,131071,0.041306667029857635
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,1,float16,fp8,131071,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,1,float16,float16,1,0.012122667084137598
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,1,float16,fp8,1,0.011855999628702799
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,1,float16,float16,3,0.012991999586423239
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,1,float16,fp8,3,0.012128000458081564
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,1,float16,float16,7,0.011722666521867117
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,1,float16,fp8,7,0.0116799995303154
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,1,float16,float16,15,0.012479999413092932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,1,float16,fp8,15,0.011727999895811081
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,1,float16,float16,31,0.011429333438475927
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,1,float16,fp8,31,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,1,float16,float16,63,0.011407999942700068
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,1,float16,fp8,63,0.012965332716703415
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,1,float16,float16,127,0.011792000383138657
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,1,float16,fp8,127,0.011706666400035223
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,1,float16,float16,255,0.011605333536863327
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,1,float16,fp8,255,0.013855999956528345
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,1,float16,float16,511,0.012351999680201212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,1,float16,fp8,511,0.01332266628742218
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,1,float16,float16,1023,0.012853333105643591
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,1,float16,fp8,1023,0.017386666188637417
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,1,float16,float16,2047,0.013797332843144735
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,1,float16,fp8,2047,0.012944000462690989
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,1,float16,float16,4095,0.012730666746695837
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,1,float16,fp8,4095,0.014175999909639359
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,1,float16,float16,8191,0.016810666769742966
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,1,float16,fp8,8191,0.01870399961868922
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,1,float16,float16,16383,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,1,float16,fp8,16383,0.02309866746266683
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,1,float16,float16,32767,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,1,float16,fp8,32767,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,1,float16,float16,65535,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,1,float16,fp8,65535,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,1,float16,float16,131071,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,1,float16,float16,1,0.010863999525705973
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,1,float16,fp8,131071,0.03311466674009959
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,1,float16,fp8,1,0.011674666156371435
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,1,float16,float16,3,0.011952000359694162
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,1,float16,fp8,3,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,1,float16,float16,7,0.01137599969903628
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,1,float16,fp8,7,0.010575999816258749
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,1,float16,float16,15,0.01099733387430509
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,1,float16,fp8,15,0.01240533341964086
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,1,float16,float16,31,0.010527999450763067
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,1,float16,float16,63,0.011685332904259363
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,1,float16,fp8,63,0.011039999624093374
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,1,float16,float16,127,0.012250666817029318
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,1,float16,fp8,127,0.01109333336353302
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,1,float16,float16,255,0.012335999558369318
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,1,float16,fp8,255,0.011029332876205444
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,1,float16,float16,511,0.012213333199421564
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,1,float16,fp8,511,0.012554666648308435
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,1,float16,float16,1023,0.012618667135636011
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,1,float16,fp8,1023,0.012576000144084295
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,1,float16,float16,2047,0.012736000120639801
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,1,float16,fp8,2047,0.012879999975363413
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,1,float16,float16,4095,0.012901333471139273
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,1,float16,fp8,4095,0.012896000097195307
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,1,float16,float16,8191,0.01869333287080129
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,1,float16,fp8,8191,0.01870399961868922
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,1,float16,float16,16383,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,1,float16,fp8,16383,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,1,float16,float16,32767,0.028336000939210255
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,1,float16,fp8,32767,0.02867199977238973
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,1,float16,float16,65535,0.031061333914597828
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,1,float16,fp8,65535,0.03071466585000356
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,1,float16,float16,131071,0.035162667433420815
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,1,float16,fp8,131071,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,1,float16,float16,1,0.012282667060693106
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,1,float16,fp8,1,0.011258666714032492
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,1,float16,float16,3,0.01228800043463707
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,1,float16,fp8,3,0.010687999427318573
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,1,float16,float16,7,0.012458667159080505
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,1,float16,fp8,7,0.010858666151762009
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,1,float16,float16,15,0.012351999680201212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,1,float16,fp8,15,0.011770666887362799
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,1,float16,float16,31,0.011930666863918304
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,1,float16,fp8,31,0.011071999867757162
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,1,float16,float16,63,0.011472000430027643
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,1,float16,fp8,63,0.010858666151762009
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,1,float16,float16,127,0.01240533341964086
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,1,float16,fp8,127,0.010746666540702185
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,1,float16,float16,255,0.012005332857370377
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,1,float16,fp8,255,0.012304000556468964
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,1,float16,float16,511,0.012410666793584824
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,1,float16,fp8,511,0.013429333766301474
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,1,float16,float16,1023,0.013050666699806849
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,1,float16,fp8,1023,0.012629333883523941
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,1,float16,float16,2047,0.014650666465361914
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,1,float16,fp8,2047,0.013082666943470636
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,1,float16,float16,4095,0.012906666845083237
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,1,float16,fp8,4095,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,1,float16,float16,8191,0.01894933357834816
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,1,float16,fp8,8191,0.020421333611011505
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,1,float16,float16,16383,0.022800001005331676
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,1,float16,fp8,16383,0.022874665757020313
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,1,float16,float16,32767,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,1,float16,fp8,32767,0.022874665757020313
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,1,float16,float16,65535,0.027989332874615986
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,1,float16,fp8,65535,0.02696000039577484
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,1,float16,float16,131071,0.04539733131726583
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,1,float16,fp8,131071,0.03242666771014532
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,1,float16,float16,1,0.011936000237862269
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,1,float16,fp8,1,0.011434666812419891
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,1,float16,float16,3,0.011861333002646765
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,1,float16,fp8,3,0.01239466667175293
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,1,float16,float16,7,0.012319999436537424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,1,float16,fp8,7,0.011264000087976456
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,1,float16,float16,15,0.012319999436537424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,1,float16,fp8,15,0.011365332951148352
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,1,float16,float16,31,0.012282667060693106
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,1,float16,fp8,31,0.011440000186363855
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,1,float16,float16,63,0.0122079998254776
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,1,float16,fp8,63,0.011440000186363855
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,1,float16,float16,127,0.0129120002190272
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,1,float16,fp8,127,0.011776000261306763
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,1,float16,float16,255,0.012080000092585882
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,1,float16,fp8,255,0.01109333336353302
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,1,float16,float16,511,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,1,float16,fp8,511,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,1,float16,float16,1023,0.012741333494583765
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,1,float16,fp8,1023,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,1,float16,float16,2047,0.018511999398469925
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,1,float16,fp8,2047,0.016656000167131424
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,1,float16,float16,4095,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,1,float16,fp8,4095,0.02075200031201045
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,1,float16,float16,8191,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,1,float16,fp8,8191,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,1,float16,float16,16383,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,1,float16,fp8,16383,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,1,float16,float16,15,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,1,float16,float16,32767,0.10308266679445903
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,1,float16,fp8,32767,0.06041066845258077
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,1,float16,float16,65535,0.18278400103251138
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,1,float16,fp8,65535,0.10240000486373901
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,1,float16,float16,1,0.012741333494583765
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,1,float16,fp8,1,0.012367999802033106
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,1,float16,float16,3,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,1,float16,fp8,3,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,1,float16,float16,7,0.012175999581813812
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,1,float16,fp8,7,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,1,float16,float16,15,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,1,float16,float16,31,0.013408000270525614
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,1,float16,fp8,15,0.013674666484196981
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,1,float16,fp8,31,0.011946666985750198
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,1,float16,float16,63,0.011749333391586939
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,1,float16,fp8,63,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,1,float16,float16,127,0.011621333658695221
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,1,float16,fp8,127,0.011829332758982977
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,1,float16,float16,255,0.012346666306257248
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,1,float16,fp8,255,0.012437333663304647
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,1,float16,float16,511,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,1,float16,fp8,511,0.012863999853531519
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,1,float16,float16,1023,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,1,float16,fp8,1023,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,1,float16,float16,2047,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,1,float16,fp8,2047,0.02075733368595441
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,1,float16,float16,4095,0.03857066730658213
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,1,float16,fp8,4095,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,1,float16,float16,8191,0.06075733403364817
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,1,float16,fp8,8191,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,1,float16,float16,16383,0.10274133086204529
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,1,float16,fp8,16383,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,1,float16,float16,32767,0.1851573387781779
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,1,float16,fp8,32767,0.10478933652242024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,1,float16,float16,65535,0.35140268007914227
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,1,float16,float16,1,0.012042666474978128
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,1,float16,fp8,1,0.012309333930412928
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,1,float16,fp8,65535,0.18858667214711508
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,1,float16,float16,3,0.012549333274364471
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,1,float16,fp8,3,0.012752000242471695
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,1,float16,float16,7,0.012042666474978128
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,1,float16,fp8,7,0.01192533348997434
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,1,float16,float16,15,0.01163200040658315
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,1,float16,fp8,15,0.011994666109482447
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,1,float16,float16,31,0.012298667182525
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,1,float16,fp8,31,0.01209066684047381
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,1,float16,float16,63,0.011754666765530905
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,1,float16,fp8,63,0.012560000022252401
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,1,float16,float16,127,0.01232533281048139
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,1,float16,fp8,127,0.011359999577204386
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,1,float16,float16,255,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,1,float16,fp8,255,0.011941333611806234
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,1,float16,float16,511,0.012266666938861212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,1,float16,fp8,511,0.013818666338920593
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,1,float16,float16,1023,0.013429333766301474
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,1,float16,fp8,1023,0.011973333855470022
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,1,float16,float16,2047,0.014394666999578476
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,1,float16,fp8,2047,0.014607999473810196
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,1,float16,float16,4095,0.014757333944241205
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,1,float16,fp8,4095,0.014912000546852747
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,1,float16,float16,8191,0.020661332954963047
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,1,float16,fp8,8191,0.02252800017595291
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,1,float16,float16,16383,0.02492266645034154
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,1,float16,fp8,16383,0.022815999885400135
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,1,float16,float16,32767,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,1,float16,fp8,32767,0.02515200028816859
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,1,float16,float16,65535,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,1,float16,fp8,65535,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,1,float16,float16,131071,0.04778666794300079
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,1,float16,float16,1,0.012074666718641916
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,1,float16,fp8,131071,0.034474665919939675
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,1,float16,fp8,1,0.012240000069141388
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,1,float16,float16,3,0.011968000481526056
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,1,float16,fp8,3,0.012373333175977072
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,1,float16,float16,7,0.012037333101034164
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,1,float16,fp8,7,0.011957333733638128
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,1,float16,float16,15,0.012576000144084295
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,1,float16,fp8,15,0.013007999708255133
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,1,float16,float16,31,0.011237333218256632
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,1,float16,fp8,31,0.013306666165590286
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,1,float16,float16,63,0.013749333719412485
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,1,float16,fp8,63,0.011546666423479715
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,1,float16,fp8,127,0.012175999581813812
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,1,float16,float16,255,0.011744000017642975
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,1,float16,fp8,255,0.010821333775917688
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,1,float16,fp8,511,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,1,float16,float16,1023,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,1,float16,fp8,1023,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,1,float16,float16,2047,0.014570667097965876
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,1,float16,fp8,2047,0.014757333944241205
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,1,float16,float16,4095,0.018677332748969395
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,1,float16,fp8,4095,0.01868266612291336
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,1,float16,float16,8191,0.0210506667693456
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,1,float16,fp8,8191,0.021045332153638203
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,1,float16,float16,16383,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,1,float16,fp8,16383,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,1,float16,float16,32767,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,1,float16,fp8,32767,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,1,float16,float16,65535,0.041306667029857635
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,1,float16,fp8,65535,0.029018667836983997
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,1,float16,float16,131071,0.06382933259010315
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,1,float16,fp8,131071,0.044026667873064675
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,1,float16,float16,1,0.013023999830087027
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,1,float16,fp8,1,0.01268799975514412
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,1,float16,float16,3,0.012506666282812754
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,1,float16,fp8,3,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,1,float16,float16,7,0.012341332932313284
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,1,float16,fp8,7,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,1,float16,float16,15,0.01268799975514412
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,1,float16,fp8,15,0.012351999680201212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,1,float16,float16,31,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,1,float16,fp8,31,0.012357333054145178
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,1,float16,float16,63,0.01268799975514412
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,1,float16,fp8,63,0.012362666428089142
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,1,float16,float16,127,0.01268799975514412
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,1,float16,fp8,127,0.012693333129088083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,1,float16,float16,255,0.012863999853531519
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,1,float16,fp8,255,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,1,float16,float16,511,0.012693333129088083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,1,float16,fp8,511,0.01250133290886879
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,1,float16,float16,1023,0.015077333897352219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,1,float16,fp8,1023,0.014746667196353277
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,1,float16,float16,2047,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,1,float16,fp8,2047,0.02072000006834666
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,1,float16,float16,4095,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,1,float16,fp8,4095,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,1,float16,float16,8191,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,1,float16,fp8,8191,0.04095999896526337
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,1,float16,float16,16383,0.10308266679445903
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,1,float16,fp8,16383,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,1,float16,float16,32767,0.18517333269119263
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,1,float16,fp8,32767,0.10478933652242024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,1,float16,float16,1,0.014394666999578476
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,1,float16,fp8,1,0.014405333747466406
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,1,float16,float16,3,0.01402666668097178
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,1,float16,fp8,3,0.014570667097965876
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,1,float16,float16,7,0.01422400027513504
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,1,float16,fp8,7,0.013376000026861826
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,1,float16,float16,15,0.014741333822409311
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,1,float16,fp8,15,0.01302933320403099
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,1,float16,float16,31,0.014576000471909841
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,1,float16,fp8,31,0.01303999995191892
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,1,float16,float16,63,0.014570667097965876
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,1,float16,fp8,63,0.014405333747466406
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,1,float16,float16,127,0.014384000251690546
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,1,float16,fp8,127,0.014565333724021912
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,1,float16,float16,255,0.014554666976133982
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,1,float16,fp8,31,0.01249066616098086
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,1,float16,fp8,255,0.015210667004187902
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,1,float16,float16,511,0.016810666769742966
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,1,float16,fp8,511,0.015072000523408255
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,1,float16,float16,1023,0.020879998803138733
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,1,float16,fp8,1023,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,1,float16,float16,2047,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,1,float16,fp8,2047,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,1,float16,float16,4095,0.06211733321348826
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,1,float16,fp8,4095,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,1,float16,float16,8191,0.10342400272687276
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,1,float16,fp8,8191,0.06382399797439575
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,1,float16,float16,16383,0.18824533621470133
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,1,float16,fp8,16383,0.10785067081451416
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,1,float16,float16,32767,0.35447998841603595
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,1,float16,fp8,32767,0.19166400035222372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,1,float16,float16,1,0.014618666221698126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,1,float16,fp8,1,0.0144213338692983
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,1,float16,float16,3,0.014757333944241205
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,1,float16,fp8,3,0.014682666709025701
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,1,float16,float16,7,0.01462399959564209
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,1,float16,fp8,7,0.014565333724021912
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,1,float16,float16,15,0.014607999473810196
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,1,float16,fp8,15,0.014576000471909841
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,1,float16,float16,31,0.014570667097965876
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,1,float16,fp8,31,0.013749333719412485
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,1,float16,float16,63,0.014442666123310724
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,1,float16,fp8,63,0.013034666577974955
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,1,float16,float16,127,0.014576000471909841
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,1,float16,fp8,127,0.012853333105643591
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,1,float16,float16,255,0.014570667097965876
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,1,float16,fp8,255,0.014560000350077948
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,1,float16,float16,511,0.01695999999841054
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,1,float16,fp8,511,0.014645333091417948
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,1,float16,float16,1023,0.020831999679406483
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,1,float16,fp8,1023,0.018672000616788864
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,1,float16,float16,2047,0.04028266668319702
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,1,float16,fp8,2047,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,1,float16,float16,4095,0.06348800162474315
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,1,float16,fp8,4095,0.04265599946180979
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,1,float16,float16,8191,0.10513066252072652
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,1,float16,fp8,8191,0.06451199948787689
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,1,float16,float16,1,0.01695999999841054
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,1,float16,fp8,1,0.015098666151364645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,1,float16,float16,3,0.01695466662446658
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,1,float16,fp8,3,0.015450666348139444
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,1,float16,float16,7,0.01695999999841054
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,1,float16,fp8,7,0.014906667172908783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,1,float16,float16,15,0.016666666915019352
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,1,float16,fp8,15,0.014762666076421738
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,1,float16,float16,31,0.01681600014368693
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,1,float16,fp8,31,0.014912000546852747
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,1,float16,float16,63,0.01695999999841054
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,1,float16,fp8,63,0.014570667097965876
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,1,float16,float16,127,0.01681600014368693
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,1,float16,fp8,127,0.016143999993801117
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,1,float16,float16,255,0.016597333053747814
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,1,float16,fp8,255,0.015360000232855478
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,1,float16,float16,511,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,1,float16,fp8,511,0.01905599981546402
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,1,float16,float16,1023,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,1,float16,fp8,1023,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,1,float16,float16,2047,0.06211733321348826
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,1,float16,fp8,2047,0.04232533276081085
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,1,float16,float16,4095,0.10478933652242024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,1,float16,fp8,4095,0.06451199948787689
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,1,float16,float16,8191,0.1889280080795288
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,1,float16,fp8,8191,0.10683733224868774
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,1,float16,float16,1,0.016965333372354507
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,1,float16,fp8,1,0.014741333822409311
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,1,float16,float16,3,0.016810666769742966
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,1,float16,fp8,3,0.016469333320856094
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,1,float16,float16,7,0.01695466662446658
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,1,float16,fp8,7,0.014917333920796713
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,1,float16,float16,15,0.01695999999841054
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,1,float16,fp8,15,0.016613333175579708
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,1,float16,float16,31,0.016943999876578648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,1,float16,fp8,31,0.01646399994691213
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,1,float16,float16,63,0.016965333372354507
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,1,float16,fp8,63,0.01647466669480006
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,1,float16,float16,127,0.016789333273967106
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,1,float16,fp8,127,0.016623999923467636
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,1,float16,float16,255,0.01672533278663953
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,1,float16,fp8,255,0.016623999923467636
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,1,float16,float16,511,0.02184533327817917
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,1,float16,fp8,511,0.019007999449968338
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,1,float16,float16,1023,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,1,float16,fp8,1023,0.02458133300145467
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,1,float16,float16,2047,0.0631466656923294
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,1,float16,fp8,2047,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,1,float16,float16,4095,0.10547199845314026
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,1,float16,fp8,4095,0.06485333542029063
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,1,float16,float16,1,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,1,float16,fp8,1,0.019002666076024372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,1,float16,float16,3,0.02276266614596049
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,1,float16,fp8,3,0.020037333170572918
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,1,float16,float16,7,0.022874665757020313
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,1,float16,fp8,7,0.02070933332045873
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,1,float16,float16,15,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,1,float16,fp8,15,0.01934933289885521
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,1,float16,float16,31,0.022863999009132385
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,1,float16,fp8,31,0.019685332973798115
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,1,float16,float16,63,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,1,float16,fp8,63,0.019685332973798115
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,1,float16,float16,127,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,1,float16,fp8,127,0.01933866615096728
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,1,float16,float16,255,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,1,float16,fp8,255,0.018661333868900936
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,1,float16,float16,511,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,1,float16,fp8,511,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,1,float16,float16,1023,0.05358933409055074
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,1,float16,fp8,1023,0.03549866626660029
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,1,float16,float16,2047,0.09864532947540283
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,1,float16,fp8,2047,0.060421332716941833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,1,float16,float16,4095,0.17834667364756265
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,1,float16,fp8,4095,0.09864532947540283
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,1,float16,float16,1,0.011973333855470022
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,1,float16,fp8,1,0.01268799975514412
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,1,float16,float16,3,0.01109333336353302
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,1,float16,fp8,3,0.012165332833925882
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,1,float16,float16,7,0.010816000401973724
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,1,float16,fp8,7,0.011893333246310553
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,1,float16,float16,15,0.010650667051474253
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,1,float16,fp8,15,0.012421333541472753
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,1,float16,fp8,31,0.011168000598748526
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,1,float16,float16,63,0.010805333654085795
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,1,float16,float16,127,0.011973333855470022
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,1,float16,fp8,63,0.012250666817029318
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,1,float16,float16,127,0.012517333030700684
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,1,float16,fp8,127,0.012960000584522883
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,1,float16,float16,255,0.012608000387748083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,1,float16,float16,511,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,1,float16,fp8,255,0.013594667116800943
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,1,float16,float16,511,0.012970666090647379
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,1,float16,fp8,511,0.012618667135636011
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,1,float16,float16,1023,0.012351999680201212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,1,float16,fp8,1023,0.013573333621025085
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,1,float16,float16,2047,0.014677333335081736
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,1,float16,fp8,2047,0.014912000546852747
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,1,float16,float16,4095,0.01870399961868922
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,1,float16,fp8,4095,0.01870399961868922
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,1,float16,float16,8191,0.020901332298914593
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,1,float16,fp8,8191,0.02089066555102666
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,1,float16,float16,16383,0.022874665757020313
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,1,float16,fp8,16383,0.02276800076166789
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,1,float16,float16,32767,0.024853333830833435
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,1,float16,fp8,32767,0.02457600086927414
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,1,float16,float16,65535,0.043007999658584595
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,1,float16,fp8,65535,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,1,float16,float16,131071,0.06417066852251689
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,1,float16,fp8,131071,0.044026667873064675
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,1,float16,float16,1,0.012752000242471695
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,1,float16,fp8,1,0.011936000237862269
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,1,float16,float16,3,0.010581333190202713
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,1,float16,fp8,3,0.012768000364303589
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,1,float16,float16,7,0.012223999947309494
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,1,float16,fp8,7,0.012266666938861212
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,1,float16,float16,15,0.01232533281048139
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,1,float16,fp8,15,0.01180800050497055
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,1,float16,float16,31,0.012191999703645706
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,1,float16,fp8,31,0.013584000368913015
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,1,float16,float16,63,0.012106666962305704
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,1,float16,fp8,63,0.012960000584522883
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,1,float16,float16,127,0.012602667013804117
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,1,float16,fp8,127,0.011893333246310553
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,1,float16,float16,255,0.012378666549921036
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,1,float16,fp8,255,0.01303999995191892
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,1,float16,float16,511,0.012400000045696894
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,1,float16,fp8,511,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,1,float16,float16,1023,0.0136266661187013
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,1,float16,fp8,1023,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,1,float16,float16,2047,0.016607999801635742
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,1,float16,fp8,2047,0.01666133354107539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,1,float16,fp8,4095,0.0186666672428449
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,1,float16,float16,8191,0.020762667059898376
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,1,float16,fp8,8191,0.020725333442290623
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,1,float16,float16,16383,0.024847999215126038
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,1,float16,fp8,16383,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,1,float16,float16,32767,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,1,float16,fp8,32767,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,1,float16,float16,65535,0.06075733403364817
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,1,float16,fp8,65535,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,1,float16,float16,131071,0.10274666547775269
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,1,float16,fp8,131071,0.062463998794555664
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,1,float16,float16,1,0.022821334501107533
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,1,float16,fp8,1,0.02070933332045873
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,1,float16,float16,3,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,1,float16,fp8,3,0.01933866615096728
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,1,float16,float16,7,0.022805333137512207
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,1,float16,fp8,7,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,1,float16,float16,15,0.022805333137512207
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,1,float16,fp8,15,0.01939733326435089
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,1,float16,float16,31,0.022800001005331676
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,1,float16,fp8,31,0.018863999595244724
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,1,float16,float16,63,0.022800001005331676
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,1,float16,fp8,63,0.0194560003777345
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,1,float16,float16,127,0.022800001005331676
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,1,float16,fp8,127,0.019007999449968338
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,1,float16,float16,255,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,1,float16,fp8,255,0.019018666197856266
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,1,float16,float16,511,0.03346133232116699
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,1,float16,fp8,511,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,1,float16,float16,1023,0.05530133346716563
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,1,float16,fp8,1023,0.037205333511034645
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,1,float16,float16,2047,0.10035199920336406
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,1,float16,fp8,2047,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,1,float16,float16,1,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,1,float16,fp8,1,0.02697066714366277
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,1,float16,float16,3,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,1,float16,fp8,3,0.028325334191322327
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,1,float16,float16,7,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,1,float16,fp8,7,0.02697066714366277
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,1,float16,fp8,15,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,1,float16,float16,15,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,1,float16,float16,31,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,1,float16,fp8,31,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,1,float16,float16,63,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,1,float16,fp8,63,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,1,float16,float16,127,0.03310399999221166
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,1,float16,fp8,127,0.026890667776266735
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,1,float16,float16,255,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,1,float16,fp8,255,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,1,float16,fp8,511,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,1,float16,float16,1023,0.09454933802286784
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,1,float16,fp8,1023,0.05904000004132589
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,1,float16,float16,2047,0.1790293256441752
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,1,float16,fp8,2047,0.10274133086204529
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,1,float16,float16,1,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,1,float16,fp8,1,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,1,float16,float16,3,0.03311999887228012
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,1,float16,fp8,3,0.028666667640209198
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,1,float16,float16,7,0.03310933212439219
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,1,float16,fp8,7,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,1,float16,float16,15,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,1,float16,fp8,15,0.02867199977238973
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,1,float16,float16,31,0.034815999368826546
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,1,float16,fp8,31,0.029018667836983997
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,1,float16,float16,63,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,1,float16,fp8,63,0.027189334233601887
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,1,float16,float16,127,0.03345066557327906
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,1,float16,fp8,127,0.028677334388097126
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,1,float16,float16,255,0.03515733281771342
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,1,float16,fp8,255,0.0290133332212766
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,1,float16,float16,511,0.05597866574923197
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,1,float16,fp8,511,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,1,float16,float16,1023,0.09523199995358785
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,1,float16,fp8,1023,0.05972800155480703
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,1,float16,float16,1,0.0576853354771932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,1,float16,fp8,1,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,1,float16,float16,3,0.05598933498064677
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,1,float16,fp8,3,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,1,float16,float16,7,0.05563733478387197
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,1,float16,fp8,7,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,1,float16,float16,15,0.0576853354771932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,1,float16,fp8,15,0.04334400097529093
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,1,float16,float16,31,0.0576853354771932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,1,float16,fp8,31,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,1,float16,float16,63,0.0576853354771932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,1,float16,fp8,63,0.04334933559099833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,1,float16,float16,127,0.056320001681645714
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,1,float16,fp8,127,0.04404266675313314
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,1,float16,float16,255,0.0576853354771932
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,1,float16,fp8,255,0.04539733131726583
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,1,float16,float16,511,0.09489066402117412
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,1,float16,fp8,511,0.06621333460013072
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,1,float16,float16,31,0.01098666712641716
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,1,float16,float16,1023,0.17254400253295898
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,1,float16,float16,1,0.012543999900420507
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,1,float16,fp8,1023,0.10172266761461894
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,1,float16,fp8,1,0.012378666549921036
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,1,float16,float16,3,0.011765333513418833
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,1,float16,fp8,3,0.012213333199421564
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,1,float16,float16,7,0.01098666712641716
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,1,float16,fp8,7,0.012122667084137598
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,1,float16,float16,15,0.011509332805871964
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,1,float16,fp8,15,0.012053333222866058
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,1,float16,float16,31,0.01098666712641716
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,1,float16,fp8,31,0.011994666109482447
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,1,float16,float16,63,0.010981333752473196
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,1,float16,float16,127,0.011519999553759893
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,1,float16,fp8,127,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,1,float16,float16,255,0.011839999506870905
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,1,float16,fp8,255,0.012533333152532578
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,1,float16,float16,511,0.012714666624863943
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,1,float16,fp8,511,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,1,float16,float16,1023,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,1,float16,fp8,1023,0.012693333129088083
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,1,float16,float16,2047,0.016810666769742966
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,1,float16,fp8,2047,0.01695999999841054
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,1,float16,float16,4095,0.019007999449968338
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,1,float16,fp8,4095,0.019002666076024372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,1,float16,float16,8191,0.020821332931518555
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,1,float16,float16,4095,0.01870399961868922
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,1,float16,fp8,8191,0.020831999679406483
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,1,float16,float16,16383,0.026965332527955372
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,1,float16,fp8,16383,0.022869333624839783
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,1,float16,float16,32767,0.04130133241415024
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,1,float16,fp8,32767,0.02697066714366277
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,1,float16,float16,65535,0.06178133189678192
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,1,float16,fp8,65535,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,1,float16,float16,131071,0.10273067156473796
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,1,float16,fp8,131071,0.06348800162474315
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,1,float16,float16,1,0.010512000570694605
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,1,float16,fp8,1,0.010981333752473196
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,1,float16,float16,3,0.011034666250149408
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,1,float16,fp8,3,0.012522666404644648
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,1,float16,float16,7,0.011584000041087469
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,1,float16,fp8,7,0.012624000509579977
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,1,float16,float16,15,0.010863999525705973
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,1,float16,fp8,15,0.012527999778588613
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,1,float16,float16,31,0.010858666151762009
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,1,float16,float16,63,0.010698666175206503
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,1,float16,fp8,63,0.010682666053374609
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,1,float16,float16,127,0.010687999427318573
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,1,float16,fp8,127,0.010512000570694605
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,1,float16,float16,255,0.010512000570694605
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,1,float16,fp8,255,0.010527999450763067
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,1,float16,float16,511,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,1,float16,fp8,511,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,1,float16,float16,1023,0.01257066677014033
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,1,float16,fp8,1023,0.012565333396196365
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,1,float16,float16,2047,0.01666133354107539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,1,float16,fp8,2047,0.01666133354107539
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,1,float16,float16,4095,0.017349333812793095
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,1,float16,fp8,4095,0.01655999943614006
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,1,float16,float16,8191,0.022863999009132385
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,1,float16,fp8,8191,0.01877333347996076
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,1,float16,float16,16383,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,1,float16,fp8,16383,0.02491733431816101
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,1,float16,float16,32767,0.05973333120346069
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,1,float16,fp8,32767,0.03925333420435587
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,1,float16,float16,65535,0.10103467106819153
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,1,float16,fp8,65535,0.05905599892139435
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,1,float16,float16,131071,0.18551466862360635
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,1,float16,fp8,131071,0.10069866975148518
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,1,float16,float16,511,0.05563733478387197
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,1,float16,fp8,63,0.011882666498422623
TRTLLM,1.0.0rc6,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,1,float16,fp8,31,0.012538666526476542
