framework,version,device,op_name,kernel_source,batch_size,isl,num_heads,num_key_value_heads,head_dim,window_size,beam_width,attn_dtype,kv_cache_dtype,step,latency
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,3,0.03874133278926214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,7,0.03976533313592275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,1,0.03737599899371465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,15,0.04983466863632202
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,63,0.056661332647005715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,31,0.05034666756788889
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,255,0.12168533603350322
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,1023,0.6253226598103842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,511,0.3199999928474426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,127,0.0769706666469574
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,2047,1.2339200178782146
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,3,0.011434666812419891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,4095,2.4432640075683594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,15,0.011605333536863327
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,31,0.012458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,255,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,511,0.03362133353948593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,1023,0.03908266623814901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,2047,0.05717333157857259
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,4095,0.1181013286113739
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,8191,0.21248000860214233
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,1,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,16383,0.41096532344818115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,3,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,7,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,15,0.019968000551064808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,127,0.02628266563018163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,63,0.02611200014750163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,31,0.021674667795499165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,255,0.0264533335963885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,511,0.03054933249950409
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,1023,0.04642133414745331
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,2047,0.10598400235176086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,8191,0.3781973520914714
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,4095,0.19797333081563315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,1,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,16383,0.7475199699401855
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,3,0.0730453332265218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,15,0.09403733412424724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,7,0.07492266595363617
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,63,0.10598400235176086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,127,0.1389226714769999
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,31,0.10188800096511841
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,255,0.2300586700439453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,511,0.6275413433710734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,1023,1.235968033472697
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,1,0.017237332959969837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,2047,2.4401920636494956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,3,0.017407999684413273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,7,0.01757866640885671
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,15,0.017749333133300144
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,127,0.02065066620707512
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,31,0.020309332758188248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,63,0.020309332758188248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,1023,0.09693866968154907
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,511,0.036864000062147774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,2047,0.17612799008687338
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,4095,0.35123201211293537
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,8191,0.6925653616587321
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,16383,1.4187520345052083
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,1,0.13431466619173685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,3,0.13943466544151306
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,7,0.15035733580589294
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,15,0.19285333156585693
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,31,0.19694934288660684
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,127,0.2635093331336975
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,63,0.20309333006540933
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,255,0.4379306634267171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,511,1.2467199961344402
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,1023,2.4581119219462075
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,1,0.26897066831588745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,3,0.29320534070332843
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,7,0.31061333417892456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,15,0.37887998421986896
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,31,0.3867306709289551
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,63,0.39509332180023193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,127,0.5143893162409464
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,255,0.850261370340983
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,float16,1,0.5720746517181396
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,float16,3,0.5908480087916056
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,float16,15,0.7507627010345459
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,float16,7,0.6142293214797974
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,1,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,3,0.016554666062196095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,float16,31,0.7661226590474447
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,float16,63,0.7801173528035482
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,float16,127,1.01256529490153
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,15,0.01791999985774358
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,7,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,31,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,63,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,127,0.021333334346612293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,255,0.03328000009059906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,511,0.09557333588600159
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,1023,0.17800533771514893
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,2047,0.3505493402481079
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,4095,0.6862506866455078
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,8191,1.24945068359375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,16383,3.6384426752726235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,float16,1,1.137664000193278
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,float16,3,1.179477294286092
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,float16,7,1.222826639811198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,float16,31,1.5209813117980957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,float16,15,1.4981120427449544
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,float16,63,1.5645012855529785
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,1,0.023039999107519787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,3,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,7,0.023893333971500397
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,float16,1,2.269866625467936
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,float16,3,2.3459839820861816
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,15,0.027818667391935985
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,31,0.027989332874615986
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,63,0.028330666323502857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,255,0.07167999943097432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,127,0.04351999859015147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,float16,7,2.431829293568929
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,511,0.16639999548594156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,float16,15,2.9818881352742515
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,float16,31,3.0431572596232095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,1023,0.35174401601155597
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,2047,0.6306133270263672
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,4095,1.240063985188802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,8191,2.5536853472391763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,3,0.031061333914597828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,7,0.032085334261258446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,15,0.039594667653242745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,1,0.030037333567937214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,31,0.039594667653242745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,63,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,127,0.06109866499900818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,511,0.2501973311106364
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,255,0.09915733337402344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,1023,0.46967466672261554
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,2047,0.9239892959594727
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,31,0.012458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,63,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,4095,1.8297173182169597
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,255,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,511,0.020309332758188248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,8191,0.15411200126012167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,1023,0.03583999971548716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,2047,0.04642133414745331
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,4095,0.08755200107892354
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,16383,0.28706133365631104
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,7,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,1,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,31,0.012458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,255,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,511,0.034304000437259674
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,2047,0.06911999980608623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,1023,0.039936001102129616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,8191,0.2599253257115682
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,16383,0.49373865127563477
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,1,0.0554666668176651
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,3,0.0576853354771932
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,7,0.05870933334032694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,4095,0.1423360009988149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,15,0.07236266632874806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,31,0.07406933108965556
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,63,0.08243200182914734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,127,0.10871466994285583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,511,0.47223468621571857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,1023,0.9277439912160238
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,1,0.010410666465759277
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,7,0.010751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,3,0.010751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,31,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,63,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,2047,1.8385920524597168
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,15,0.011776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,255,0.17885865767796835
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,255,0.016554666062196095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,2047,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,1023,0.06724266707897186
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,511,0.0240639994541804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,4095,0.24064000447591147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,16383,0.9859413305918375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,8191,0.46694401899973553
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,3,0.10700800021489461
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,7,0.11110400160153706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,15,0.14455466469128928
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,31,0.15018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,63,0.1546239952246348
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,127,0.20241065820058188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,1,0.10308266679445903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,511,0.9308160146077474
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,1023,1.836202621459961
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,1,0.19933867454528809
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,3,0.21179733673731485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,255,0.3357013463973999
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,7,0.23466666539510092
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,15,0.2860373258590698
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,63,0.2984960079193115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,127,0.39048532644907635
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,31,0.292522668838501
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,255,0.6464853286743164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,float16,1,0.43195732434590656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,float16,7,0.4648960034052531
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,float16,15,0.5655893484751383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,1,0.017749333133300144
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,float16,3,0.4466346502304077
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,float16,63,0.5872639815012614
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,float16,31,0.5763413508733114
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,3,0.018432000031073887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,15,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,7,0.018432000031073887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,31,0.024234667420387268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,127,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,255,0.0290133332212766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,float16,127,0.7674880027770996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,63,0.024234667420387268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,1023,0.12612266341845194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,511,0.05478399991989136
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,2047,0.24012800057729086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,4095,0.46865065892537433
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,8191,0.920746644337972
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,16383,1.969663937886556
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,float16,7,0.9190399646759033
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,float16,3,0.8872959613800049
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,float16,15,1.1272532939910889
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,float16,63,1.1644586722056072
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,float16,1,0.8584533532460531
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,float16,31,1.148586670557658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,float16,1,1.7104214032491047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,1,0.01757866640885671
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,float16,3,1.7672533988952637
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,3,0.01791999985774358
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,15,0.023210667073726654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,7,0.018602666755517323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,31,0.023210667073726654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,63,0.023210667073726654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,float16,7,1.830399990081787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,127,0.027818667391935985
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,float16,31,2.2882986068725586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,float16,15,2.2420479456583657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,255,0.053077335158983864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,511,0.12595199545224509
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,1023,0.24132267634073892
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,4095,0.9275733629862467
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,2047,0.46916266282399494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,8191,1.9230720202128093
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,1,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,7,0.02611200014750163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,3,0.025429333249727886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,15,0.03328000009059906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,31,0.03293866664171219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,127,0.05000533163547516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,255,0.08072533210118611
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,1023,0.3928746779759725
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,511,0.2034346659978231
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,63,0.034304000437259674
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,4095,1.5245653788248699
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,2047,0.7703893184661865
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,15,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,7,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,31,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,63,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,255,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,511,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,1023,0.03566933423280716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,2047,0.045909335215886436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,1,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,4095,0.08499200145403545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,8191,0.14660267035166422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,16383,0.26828799645106
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,3,0.011434666812419891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,63,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,31,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,255,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,511,0.03379199902216593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,1023,0.03976533313592275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,4095,0.12475732962290446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,2047,0.06553600231806438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,8191,0.2310826579729716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,16383,0.4352000157038371
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,3,0.04693333307902018
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,1,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,7,0.048469334840774536
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,15,0.06126933296521505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,31,0.06178133189678192
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,127,0.09011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,255,0.14882133404413858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,63,0.06929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,511,0.3957759936650594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,1023,0.7748266855875651
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,2047,1.528490702311198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,3,0.010581333190202713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,1,0.010581333190202713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,7,0.010581333190202713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,15,0.011605333536863327
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,63,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,511,0.023381332556406658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,255,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,31,0.013823999712864557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,2047,0.105813334385554
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,1023,0.0554666668176651
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,4095,0.20241065820058188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,8191,0.3938986857732137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,1,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,16383,0.9233067035675049
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,3,0.08925867080688477
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,15,0.11929600437482198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,31,0.12714667121569315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,7,0.09352533022562663
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,63,0.13056000073750815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,127,0.17032533884048462
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,255,0.28381866216659546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,511,0.7787520090738932
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,1,0.16674133141835532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,1023,1.534122625986735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,3,0.17476266622543335
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,31,0.24593067169189453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,63,0.2505386670430501
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,15,0.2629973292350769
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,127,0.32819199562072754
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,7,0.19336533546447754
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,255,0.5457919836044312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,float16,1,0.3519146839777629
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,float16,3,0.374783992767334
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,1,0.01757866640885671
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,float16,7,0.38946131865183514
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,float16,15,0.47308798631032306
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,float16,31,0.48298664887746173
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,3,0.01791999985774358
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,float16,63,0.4915200074513753
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,7,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,15,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,float16,127,0.6413653294245402
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,63,0.02065066620707512
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,31,0.020821332931518555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,127,0.021333334346612293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,255,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,511,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,2047,0.20940800507863364
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,1023,0.106495996316274
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,8191,0.7743146419525146
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,16383,1.5638186136881511
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,4095,0.4026026725769043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,float16,1,0.717141310373942
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,float16,3,0.741376002629598
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,float16,7,0.7700479825337728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,float16,15,0.9391787052154541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,float16,63,0.9721173445383707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,float16,31,0.9594879945119222
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,1,0.016895999511082966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,float16,1,1.4320640563964844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,3,0.017407999684413273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,float16,3,1.4759252866109211
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,15,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,7,0.017237332959969837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,float16,7,1.5305387179056804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,31,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,63,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,float16,31,1.9087360699971516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,float16,15,1.8718719482421875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,255,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,511,0.10513066252072652
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,127,0.023381332556406658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,1023,0.20923733711242676
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,2047,0.4005546569824219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,4095,0.7862613201141357
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,8191,1.5232000350952148
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,1,0.023039999107519787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,7,0.023893333971500397
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,3,0.02372266600529353
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,15,0.028160000840822857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,127,0.04147200038035711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,1023,0.32102400064468384
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,511,0.16503467162450156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,31,0.028160000840822857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,63,0.028501334289709728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,2047,0.6301013231277466
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,255,0.06929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,4095,1.2417706648508708
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,8191,2.5511253674825034
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,31,0.012458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,7,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,63,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,255,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,511,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,1023,0.02184533327817917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,2047,0.03908266623814901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,4095,0.05819733440876007
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,8191,0.11895466844240825
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,16383,0.2135039965311686
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,32767,0.4102826515833537
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,3,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,31,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,63,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,255,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,1023,0.03925333420435587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,511,0.03345066557327906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,2047,0.057002668579419456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,4095,0.11895466844240825
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,16383,0.4097706476847331
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,1,0.03737599899371465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,32767,0.7949653466542562
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,3,0.03822933385769526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,7,0.039936001102129616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,15,0.049322664737701416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,8191,0.212991992632548
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,31,0.04966400067011515
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,127,0.07509333391984303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,63,0.05734399954477946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,255,0.12219732999801636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,511,0.3206826647122701
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,1023,0.6258346637090048
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,1,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,2047,1.2289706865946453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,3,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,7,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,15,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,31,0.021674667795499165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,63,0.02611200014750163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,4095,2.436778704325358
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,511,0.03054933249950409
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,255,0.0266239990790685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,127,0.02611200014750163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,2047,0.10734933614730835
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,1023,0.04659200211366018
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,4095,0.19780266284942627
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,8191,0.3781973520914714
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,32767,1.3410986264546711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,16383,0.741546630859375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,7,0.07509333391984303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,1,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,31,0.10257066289583842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,15,0.09471999605496724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,3,0.07338666419188182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,63,0.1063253382841746
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,127,0.13994666934013367
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,511,0.6297599871953329
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,1023,1.2364799976348877
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,2047,2.4410452842712402
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,7,0.1508693297704061
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,3,0.14045866330464682
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,1,0.13516799608866373
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,255,0.23057067394256592
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,15,0.19370667139689127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,31,0.19814399878184
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,63,0.2027519941329956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,255,0.43929600715637207
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,127,0.26521599292755127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,511,1.2472319602966309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,1,0.2701653242111206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,7,0.33126399914423627
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,3,0.2954240043958028
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,15,0.38075733184814453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,31,0.38809601465861004
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,63,0.39611732959747314
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,1,0.017407999684413273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,127,0.5178026755650839
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,3,0.01757866640885671
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,255,0.8540159861246744
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,15,0.017749333133300144
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,31,0.020138667275508244
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,7,0.017407999684413273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,63,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,127,0.020821332931518555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,255,0.024234667420387268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,511,0.03703466554482778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,2047,0.17459199825922647
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,1023,0.09557333588600159
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,4095,0.35123201211293537
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,8191,0.6901760101318359
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,16383,1.4178986549377441
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,32767,2.6103466351826987
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,float16,1,0.5766826470692953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,float16,3,0.5963093439737955
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,float16,7,0.6184959808985392
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,float16,15,0.7558826605478922
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,float16,31,0.7690239747365316
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,float16,63,0.7811413606007894
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,float16,127,1.0216106573740642
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,float16,1,1.1496106783548992
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,1,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,3,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,float16,3,1.1854506333669026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,float16,7,1.228117307027181
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,7,0.016895999511082966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,float16,15,1.5010132789611816
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,15,0.01791999985774358
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,31,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,float16,31,1.5334399541219075
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,float16,63,1.5510187149047852
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,63,0.018432000031073887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,127,0.021162666380405426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,255,0.03310933212439219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,511,0.09386666615804036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,1023,0.17919999361038208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,2047,0.3505493402481079
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,4095,0.6871039867401123
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,8191,1.257813294728597
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,16383,3.6331520080566406
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,1,0.017407999684413273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,7,0.018602666755517323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,3,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,15,0.023210667073726654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,31,0.023210667073726654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,127,0.027818667391935985
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,255,0.053077335158983864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,63,0.023381332556406658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,1023,0.2404693365097046
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,511,0.12612266341845194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,2047,0.46984533468882245
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,4095,0.9282560348510742
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,1,0.010751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,8191,1.9198293685913086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,31,0.012117333710193634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,255,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,511,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,1023,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,4095,0.04966400067011515
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,2047,0.03601066768169403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,8191,0.09949866930643718
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,16383,0.17220266660054526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,32767,0.31778132915496826
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,31,0.012458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,63,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,255,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,511,0.020309332758188248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,1023,0.03601066768169403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,2047,0.04642133414745331
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,4095,0.08738133311271667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,8191,0.1532586713631948
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,16383,0.28706133365631104
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,32767,0.5474986632664999
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,1,0.030037333567937214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,3,0.031061333914597828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,7,0.032085334261258446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,15,0.039594667653242745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,31,0.03942399968703588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,63,0.040789333482583366
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,127,0.059903999169667564
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,255,0.09591466188430786
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,511,0.2384213407834371
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,2047,0.9239892959594727
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,1023,0.4715520143508911
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,4095,1.8315946261088054
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,3,0.011434666812419891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,7,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,31,0.012629333883523941
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,15,0.011776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,127,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,511,0.034304000437259674
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,255,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,1023,0.04027733455101649
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,2047,0.07048533360163371
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,4095,0.1418239971001943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,8191,0.2606079975763957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,16383,0.49237334728240967
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,1,0.05597866574923197
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,32767,0.9594879945119222
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,3,0.0580266664425532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,31,0.07406933108965556
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,7,0.05922133227189382
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,15,0.07236266632874806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,63,0.08277333279450734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,127,0.10922666390736897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,255,0.17971199750900269
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,511,0.47377065817515057
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,1023,0.9304746786753336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,2047,1.8404693603515625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,1,0.10376532872517903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,15,0.14455466469128928
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,7,0.11178666353225708
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,3,0.10734933614730835
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,31,0.15069866180419922
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,63,0.15479466319084167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,127,0.20377600193023682
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,255,0.33826132615407306
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,1,0.1991680065790812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,511,0.928938627243042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,3,0.21248000860214233
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,7,0.23483733336130777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,15,0.2872320016225179
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,31,0.2935466567675273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,63,0.2991786599159241
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,127,0.3930453459421794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,15,0.011776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,7,0.010581333190202713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,1,0.010581333190202713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,255,0.6478506724039713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,63,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,31,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,255,0.016554666062196095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,1023,0.06758399804433186
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,511,0.0240639994541804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,2047,0.12544000148773193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,4095,0.2397866646448771
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,8191,0.4676266511281331
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,16383,0.9847466945648193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,32767,2.0520960489908853
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,float16,1,0.43263999621073407
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,float16,3,0.44817066192626953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,float16,7,0.4681386550267537
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,float16,15,0.5693440039952596
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,float16,31,0.5787306626637777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,float16,63,0.5884586572647095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,float16,127,0.7719253698984782
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,float16,1,0.86135466893514
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,1,0.01791999985774358
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,3,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,float16,3,0.8907093207041422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,7,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,float16,7,0.9245013395945231
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,float16,31,1.152341365814209
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,float16,15,1.13100798924764
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,float16,63,1.1656533082326253
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,15,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,31,0.0240639994541804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,127,0.024405332903067272
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,63,0.024405332903067272
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,255,0.02918400118748347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,511,0.05529599885145823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,2047,0.24183466037114462
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,1023,0.1276586651802063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,4095,0.4667733510335286
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,8191,0.9200639724731445
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,16383,1.969322681427002
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,1,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,3,0.016554666062196095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,15,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,7,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,127,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,63,0.018432000031073887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,255,0.03259733319282532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,31,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,511,0.09659733374913533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,1023,0.17800533771514893
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,2047,0.34884266058603924
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,8191,1.2573013305664062
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,4095,0.6855680147806803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,16383,3.6367359161376953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,7,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,15,0.011776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,31,0.012458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,63,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,511,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,255,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,1023,0.020992000897725422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,4095,0.04539733131726583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,2047,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,8191,0.06775466601053874
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,16383,0.12731732924779257
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,32767,0.23347200949986777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,65535,0.44253865877787274
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,15,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,31,0.012458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,63,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,255,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,1023,0.02184533327817917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,2047,0.03908266623814901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,511,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,4095,0.05734399954477946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,8191,0.11878400047620137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,16383,0.21128533283869425
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,32767,0.4102826515833537
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,3,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,1,0.023210667073726654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,15,0.027989332874615986
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,65535,0.7961599826812744
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,7,0.02372266600529353
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,31,0.028160000840822857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,63,0.028330666323502857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,127,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,255,0.06929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,511,0.16742400328318277
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,1023,0.3524266481399536
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,2047,0.6311253309249878
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,4095,1.2431360085805256
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,8191,2.553343931833903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,31,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,255,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,511,0.03362133353948593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,1023,0.03925333420435587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,2047,0.05649066468079885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,4095,0.1186133325099945
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,8191,0.21094399690628052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,16383,0.40994131565093994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,32767,0.7954773108164469
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,1,0.03754666695992152
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,3,0.038912000755469
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,65535,1.256277322769165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,7,0.039936001102129616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,15,0.04949333270390829
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,31,0.049322664737701416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,63,0.05717333157857259
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,127,0.07543466488520305
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,255,0.12219732999801636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,1023,0.6278826793034872
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,511,0.3198293248812358
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,2047,1.2328960100809734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,1,0.07048533360163371
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,4095,2.4378026326497397
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,3,0.07355733215808868
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,7,0.07543466488520305
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,15,0.09437867005666097
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,31,0.10257066289583842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,63,0.10683733224868774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,255,0.23125332593917847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,127,0.14079999923706055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,511,0.6263466676076254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,1,0.13482667009035745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,1023,1.2357973257700603
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,3,0.14011733730634054
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,7,0.15103999773661295
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,15,0.19421867529551187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,127,0.26709334055582684
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,31,0.1986560026804606
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,63,0.20292266209920248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,255,0.4389546712239583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,1,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,511,1.2445013523101807
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,3,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,7,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,15,0.019968000551064808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,31,0.021674667795499165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,127,0.02628266563018163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,255,0.0264533335963885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,63,0.02611200014750163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,511,0.030720000465710957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,1023,0.04642133414745331
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,2047,0.10666666428248088
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,4095,0.19660800695419312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,8191,0.37666134039560956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,16383,0.7405227025349935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,32767,1.3460480372111003
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,65535,2.5077759424845376
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,1,0.2688000003496806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,3,0.2964479923248291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,7,0.31470932563145954
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,15,0.3804159959157308
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,63,0.39611732959747314
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,31,0.3887786865234375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,127,0.5220693349838257
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,255,0.851797342300415
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,float16,1,0.5778773228327433
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,float16,3,0.5954560041427612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,float16,7,0.6227626800537109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,float16,15,0.7540053526560465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,3,0.017237332959969837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,float16,31,0.7700479825337728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,float16,63,0.7813119888305664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,float16,127,1.0292906761169434
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,7,0.01757866640885671
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,15,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,1,0.017237332959969837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,31,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,63,0.02065066620707512
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,127,0.020821332931518555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,255,0.023893333971500397
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,511,0.03669333209594091
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,1023,0.09710933764775594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,2047,0.17851734161376953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,16383,1.4230186144510906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,8191,0.6891520023345947
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,32767,2.6192213694254556
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,4095,0.35020800431569415
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,1,0.01791999985774358
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,3,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,7,0.018432000031073887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,15,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,63,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,127,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,255,0.02918400118748347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,1023,0.12612266341845194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,511,0.05495466788609823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,2047,0.24132267634073892
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,31,0.024234667420387268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,4095,0.4654080073038737
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,8191,0.9181866645812988
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,1,0.010751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,16383,1.967957337697347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,7,0.010751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,63,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,255,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,511,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,1023,0.020992000897725422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,2047,0.021162666380405426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,4095,0.02184533327817917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,31,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,8191,0.029866665601730347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,16383,0.07679999868075053
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,32767,0.1353386640548706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,65535,0.24849067131678262
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,7,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,31,0.012117333710193634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,63,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,511,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,255,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,2047,0.03618133316437403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,1023,0.021674667795499165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,4095,0.04983466863632202
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,8191,0.0993280013402303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,16383,0.1723733345667521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,32767,0.31522132953008014
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,3,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,65535,0.628053347269694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,7,0.01877333347996076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,31,0.023039999107519787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,15,0.023039999107519787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,63,0.023039999107519787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,1,0.017237332959969837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,127,0.027818667391935985
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,255,0.05341866612434387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,511,0.12544000148773193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,2047,0.47189335028330487
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,1023,0.24217599630355835
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,4095,0.9277439912160238
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,8191,1.9225600560506184
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,7,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,31,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,15,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,255,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,511,0.020309332758188248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,4095,0.08738133311271667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,2047,0.04659200211366018
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,8191,0.15359999736150107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,16383,0.2860373258590698
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,1023,0.03583999971548716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,32767,0.5481813351313273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,1,0.030037333567937214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,65535,1.0664959748586018
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,3,0.03089066594839096
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,7,0.03242666771014532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,15,0.03942399968703588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,31,0.039594667653242745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,127,0.060415998101234436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,255,0.09591466188430786
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,63,0.040789333482583366
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,511,0.23705599705378214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,1023,0.47172268231709796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,1,0.05614933371543884
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,2047,0.9270613193511963
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,4095,1.831424077351888
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,3,0.05836800237496694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,7,0.05922133227189382
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,31,0.07492266595363617
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,63,0.08277333279450734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,15,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,127,0.10956799983978271
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,255,0.17971199750900269
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,511,0.4724053144454956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,1023,0.9328640302022299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,1,0.10342400272687276
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,7,0.11195733149846394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,3,0.10769066214561462
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,31,0.15103999773661295
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,15,0.144896000623703
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,63,0.1551359991232554
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,127,0.2053119937578837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,1,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,255,0.33774932225545246
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,511,0.9308160146077474
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,31,0.012458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,255,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,511,0.034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,15,0.011605333536863327
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,1023,0.040448000033696495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,2047,0.07014399766921997
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,4095,0.1437013347943624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,8191,0.258730669816335
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,32767,0.9591466585795084
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,16383,0.4925440152486165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,65535,1.8727253278096516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,3,0.21179733673731485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,7,0.2362026572227478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,1,0.19933867454528809
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,15,0.2872320016225179
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,31,0.29422932863235474
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,63,0.2995199958483378
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,127,0.3959466616312663
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,255,0.6480213403701782
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,float16,1,0.4322986602783203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,float16,3,0.4483413298924764
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,1,0.010410666465759277
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,float16,15,0.567466656366984
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,float16,7,0.46779731909434
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,float16,63,0.5886293252309164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,float16,31,0.5802666743596395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,7,0.010751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,31,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,float16,127,0.7763626575469971
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,255,0.016554666062196095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,511,0.023893333971500397
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,63,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,1023,0.06724266707897186
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,2047,0.12595199545224509
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,4095,0.23910399278004965
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,8191,0.46557867527008057
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,16383,1.0280959606170654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,32767,2.050901254018148
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,1,0.017407999684413273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,15,0.01791999985774358
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,7,0.017749333133300144
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,3,0.017407999684413273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,31,0.020309332758188248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,63,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,127,0.02065066620707512
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,255,0.024234667420387268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,511,0.03703466554482778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,1023,0.09710933764775594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,2047,0.17578667402267456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,8191,0.6877866586049398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,16383,1.420629342397054
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,4095,0.35123201211293537
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,3,0.010751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,32767,2.6118826866149902
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,15,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,7,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,31,0.012117333710193634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,63,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,255,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,511,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,1023,0.020992000897725422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,2047,0.021162666380405426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,4095,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,8191,0.045738667249679565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,16383,0.067071999112765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,32767,0.1281706690788269
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,65535,0.23500800132751465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,1,0.010751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,15,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,131071,0.44697598616282147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,31,0.012117333710193634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,63,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,255,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,1023,0.021333334346612293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,511,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,2047,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,4095,0.045738667249679565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,8191,0.06621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,32767,0.23296000560124716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,65535,0.4413439830144246
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,16383,0.1281706690788269
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,3,0.016554666062196095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,1,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,7,0.016554666062196095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,131071,0.86954665184021
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,15,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,31,0.01791999985774358
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,63,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,255,0.03345066557327906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,1023,0.17783466974894205
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,511,0.0962559978167216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,2047,0.3529386520385742
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,4095,0.6864213148752848
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,8191,1.2557653586069744
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,127,0.021162666380405426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,16383,3.630762736002604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,31,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,255,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,511,0.019968000551064808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,1023,0.02184533327817917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,2047,0.03908266623814901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,63,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,8191,0.11929600437482198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,4095,0.05717333157857259
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,16383,0.21230934063593546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,32767,0.4094293514887492
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,65535,0.7920640309651693
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,1,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,7,0.023893333971500397
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,15,0.027989332874615986
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,131071,1.3622612953186035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,31,0.027989332874615986
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,63,0.028330666323502857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,3,0.02372266600529353
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,127,0.041984001795450844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,255,0.07014399766921997
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,511,0.16674133141835532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,1023,0.32341333230336505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,2047,0.6318080027898153
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,4095,1.2422826290130615
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,3,0.03874133278926214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,8191,2.554709275563558
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,7,0.039936001102129616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,15,0.04983466863632202
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,31,0.04966400067011515
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,1,0.03754666695992152
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,63,0.057002668579419456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,127,0.07560533285140991
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,255,0.12270933389663696
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,511,0.3203413287798564
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,1023,0.6282240152359009
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,1,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,2047,1.2339200178782146
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,7,0.07509333391984303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,15,0.09506133198738098
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,31,0.10291199882825215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,63,0.10666666428248088
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,3,0.07355733215808868
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,127,0.14079999923706055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,255,0.2307413419087728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,511,0.628053347269694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,1023,1.2398933569590251
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,7,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,31,0.012458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,15,0.011605333536863327
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,63,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,255,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,511,0.03345066557327906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,1023,0.039594667653242745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,2047,0.056832000613212585
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,4095,0.11878400047620137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,16383,0.4082346757253011
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,8191,0.21077332894007364
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,32767,0.7920640309651693
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,65535,1.2503039836883545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,3,0.14028799533843994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,1,0.1353386640548706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,7,0.1513813336690267
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,131071,2.5089707374572754
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,15,0.19353600343068442
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,63,0.20377600193023682
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,31,0.19899733861287436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,127,0.2677759925524394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,255,0.4398080110549927
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,511,1.2421120007832844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,1,0.2691413362820943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,3,0.29576534032821655
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,7,0.314026673634847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,15,0.38178133964538574
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,1,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,63,0.398848017056783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,3,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,31,0.38980265458424884
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,7,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,127,0.5222400029500326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,15,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,31,0.02184533327817917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,63,0.02628266563018163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,127,0.02611200014750163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,255,0.8538453578948975
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,511,0.03054933249950409
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,255,0.0266239990790685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,1023,0.04607999821503957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,2047,0.1063253382841746
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,4095,0.19643733898798624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,8191,0.37768534819285077
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,16383,0.7384746869405111
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,32767,1.3445119857788086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,65535,2.5053866704305015
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,1,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,3,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,15,0.019968000551064808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,7,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,63,0.02611200014750163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,127,0.02611200014750163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,255,0.0264533335963885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,31,0.02184533327817917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,511,0.031061333914597828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,1023,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,4095,0.19660800695419312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,8191,0.37495466073354083
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,16383,0.7391573588053385
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,2047,0.10769066214561462
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,32767,1.345706621805827
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,3,0.010751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,65535,2.5067520141601562
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,15,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,31,0.012117333710193634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,255,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,511,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,1023,0.020821332931518555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,63,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,2047,0.021162666380405426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,8191,0.02184533327817917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,4095,0.021333334346612293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,16383,0.0266239990790685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,32767,0.040789333482583366
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,65535,0.09403733412424724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,3,0.010751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,131071,0.17578667402267456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,7,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,31,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,15,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,63,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,511,0.019968000551064808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,255,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,1023,0.021162666380405426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,2047,0.020992000897725422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,4095,0.021674667795499165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,32767,0.15001599987347922
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,8191,0.045738667249679565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,65535,0.2326186696688334
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,1,0.01757866640885671
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,3,0.017407999684413273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,131071,0.4474879900614421
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,16383,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,7,0.017749333133300144
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,15,0.01791999985774358
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,63,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,127,0.020821332931518555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,31,0.020309332758188248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,1023,0.09727999567985535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,4095,0.3500373363494873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,2047,0.1771519978841146
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,8191,0.6894933382670084
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,16383,1.4274560610453289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,511,0.037205333511034645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,32767,2.6236586570739746
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,7,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,31,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,63,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,511,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,4095,0.045738667249679565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,2047,0.02184533327817917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,8191,0.06348800162474315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,255,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,1023,0.021162666380405426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,32767,0.2321066657702128
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,16383,0.12834133704503378
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,65535,0.4427093267440796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,3,0.016554666062196095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,1,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,31,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,15,0.01791999985774358
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,131071,0.8657920360565186
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,127,0.021333334346612293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,63,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,7,0.016554666062196095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,255,0.03259733319282532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,511,0.09676800171534221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,1023,0.1776640017827352
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,4095,0.6867626508076986
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,2047,0.35225598017374676
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,8191,1.2581546306610107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,3,0.02372266600529353
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,15,0.028160000840822857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,7,0.023893333971500397
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,16383,3.6718934377034507
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,31,0.028330666323502857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,1,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,127,0.04147200038035711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,255,0.07116800049940745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,63,0.028330666323502857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,511,0.16622933745384216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,1023,0.3232426643371582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,2047,0.6324906746546427
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,4095,1.2436479727427165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,3,0.03822933385769526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,7,0.039936001102129616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,31,0.04966400067011515
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,1,0.03737599899371465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,63,0.0576853354771932
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,15,0.04966400067011515
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,255,0.1264639993508657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,127,0.0766293356815974
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,511,0.3211946686108907
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,1023,0.626858671506246
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,2047,1.235968033472697
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,63,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,1,0.010751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,7,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,31,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,255,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,511,0.019968000551064808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,1023,0.02184533327817917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,2047,0.039594667653242745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,16383,0.21196800470352173
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,32767,0.40806400775909424
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,8191,0.11912533640861511
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,4095,0.05649066468079885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,65535,0.7944533030192057
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,1,0.07082666456699371
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,3,0.07338666419188182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,131071,1.364309310913086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,15,0.09454933802286784
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,7,0.07543466488520305
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,31,0.10291199882825215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,127,0.14250666896502176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,255,0.23756800095240274
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,63,0.10666666428248088
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,511,0.6273706754048666
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,1,0.13482667009035745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,3,0.14045866330464682
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,1023,1.248255968093872
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,7,0.16725333531697592
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,15,0.19387733936309814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,31,0.19933867454528809
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,127,0.2698240081469218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,3,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,1,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,7,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,63,0.20360533396402994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,15,0.011605333536863327
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,255,0.44919466972351074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,31,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,255,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,1023,0.03976533313592275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,511,0.03345066557327906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,2047,0.056832000613212585
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,4095,0.11946666240692139
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,511,1.2438186804453533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,8191,0.21145600080490112
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,16383,0.4065279960632324
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,32767,0.7946240107218424
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,65535,1.252522627512614
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,131071,2.512042681376139
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,7,0.07406933108965556
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,31,0.09284266829490662
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,1,0.06963199873765309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,15,0.09233066439628601
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,3,0.07236266632874806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,63,0.09403733412424724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,127,0.11212799946467082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,255,0.17459199825922647
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,511,0.307370662689209
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,1023,0.5608106851577759
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,2047,1.0803199609120686
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,4095,2.092202663421631
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,1,0.06946133573849995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,7,0.07406933108965556
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,3,0.07253333429495494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,15,0.09318400422732036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,31,0.09250133236249287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,63,0.0942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,127,0.11281067132949829
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,255,0.1776640017827352
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,1023,0.5645653406778971
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,4095,2.0787199338277182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,511,0.30498133103052777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,2047,1.0774186452229817
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,7,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,1,0.022698665658632915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,3,0.022698665658632915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,15,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,63,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,31,0.022698665658632915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,127,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,511,0.05341866612434387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,1023,0.08106666803359985
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,2047,0.11878400047620137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,255,0.03345066557327906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,4095,0.1616213321685791
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,1,0.02628266563018163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,3,0.02628266563018163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,31,0.02628266563018163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,63,0.02628266563018163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,7,0.02628266563018163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,15,0.02628266563018163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,127,0.0266239990790685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,255,0.04334933559099833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,511,0.06877866884072621
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,1023,0.11997866630554199
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,2047,0.13687466581662497
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,4095,0.20599466562271118
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,3,0.01877333347996076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,1,0.018602666755517323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,31,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,15,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,7,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,63,0.025941332181294758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,127,0.02611200014750163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,255,0.027136000494162243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,511,0.0314026673634847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,1023,0.046762665112813316
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,2047,0.0747519979874293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,4095,0.12851199507713318
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,16383,0.444757342338562
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,8191,0.23483733336130777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,1,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,3,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,7,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,31,0.021674667795499165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,63,0.025941332181294758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,127,0.02611200014750163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,15,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,511,0.031231999397277832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,1023,0.045567999283472695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,2047,0.07287466526031494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,255,0.02679466704527537
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,4095,0.12526933352152506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,8191,0.2302293380101522
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,16383,0.44066135088602704
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,1,0.01877333347996076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,3,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,15,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,7,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,31,0.02184533327817917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,63,0.025941332181294758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,255,0.0266239990790685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,127,0.025941332181294758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,511,0.03089066594839096
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,2047,0.023039999107519787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,1023,0.020821332931518555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,4095,0.025258667767047882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,8191,0.03259733319282532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,16383,0.048298666874567665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,1,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,3,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,15,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,7,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,63,0.02611200014750163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,31,0.021674667795499165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,127,0.02628266563018163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,255,0.0264533335963885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,511,0.0170666662355264
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,1023,0.01757866640885671
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,2047,0.020138667275508244
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,8191,0.03583999971548716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,16383,0.05614933371543884
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,4095,0.02918400118748347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,1,0.017407999684413273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,3,0.01757866640885671
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,7,0.01757866640885671
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,15,0.01757866640885671
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,31,0.020138667275508244
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,63,0.020309332758188248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,127,0.020992000897725422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,511,0.03498666733503342
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,255,0.025258667767047882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,1023,0.0576853354771932
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,2047,0.10103467106819153
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,4095,0.1867093245188395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,8191,0.3505493402481079
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,16383,0.7850666840871176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,1,0.0170666662355264
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,3,0.017749333133300144
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,31,0.020138667275508244
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,15,0.017749333133300144
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,7,0.017749333133300144
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,63,0.020138667275508244
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,127,0.021674667795499165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,255,0.024405332903067272
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,511,0.03515733281771342
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,1023,0.05614933371543884
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,2047,0.09915733337402344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,4095,0.18244266510009766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,8191,0.3452586730321248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,1,0.017407999684413273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,16383,0.874837319056193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,3,0.017749333133300144
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,7,0.017749333133300144
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,15,0.017749333133300144
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,31,0.020138667275508244
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,63,0.020138667275508244
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,511,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,127,0.02065066620707512
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,1023,0.020992000897725422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,2047,0.02491733431816101
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,8191,0.04471466441949209
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,4095,0.0365226666132609
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,16383,0.07116800049940745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,1,0.017237332959969837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,3,0.01757866640885671
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,15,0.01791999985774358
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,7,0.01791999985774358
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,31,0.019968000551064808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,63,0.020138667275508244
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,127,0.02065066620707512
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,255,0.016554666062196095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,511,0.0170666662355264
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,1023,0.020821332931518555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,8191,0.059392000238100685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,4095,0.039594667653242745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,16383,0.09011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,2047,0.02918400118748347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,3,0.13704533378283182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,1,0.1327786644299825
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,7,0.1430186629295349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,15,0.1786880095799764
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,31,0.1786880095799764
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,63,0.18056533734003702
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,127,0.2177706758181254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,511,0.6016000111897787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,2047,2.1038079261779785
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,1,0.13243732849756876
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,3,0.13738666971524557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,1023,1.1006293296813965
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,255,0.3438933293024699
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,15,0.1800533334414164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,7,0.14353066682815552
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,31,0.18039466937383017
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,63,0.18141865730285645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,127,0.2182826598485311
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,255,0.34781865278879803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,511,0.6068906784057617
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,2047,2.1003947257995605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,1,0.034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,3,0.034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,1023,1.1062613328297932
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,15,0.034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,7,0.034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,31,0.034645333886146545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,127,0.03379199902216593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,63,0.03362133353948593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,511,0.08703999718030293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,255,0.056320001681645714
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,1,0.04164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,3,0.041984001795450844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,2047,0.1704960068066915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,7,0.04164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,31,0.04164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,15,0.041984001795450844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,63,0.04164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,1023,0.11997866630554199
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,127,0.04181333382924398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,511,0.077824001510938
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,255,0.059903999169667564
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,1023,0.11878400047620137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,2047,0.19899733861287436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,3,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,1,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,15,0.01791999985774358
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,31,0.01791999985774358
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,63,0.018602666755517323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,127,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,255,0.029525332152843475
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,7,0.01672533278663953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,511,0.04744533201058706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,2047,0.1764693260192871
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,4095,0.3300693432490031
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,8191,0.6140586535135905
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,1023,0.09523199995358785
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,16383,2.4191999435424805
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,1,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,15,0.01791999985774358
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,7,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,63,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,31,0.01791999985774358
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,127,0.020992000897725422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,255,0.029696000119050343
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,3,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,511,0.051029334465662636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,1023,0.0846506655216217
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,2047,0.16401066382726034
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,4095,0.30344533920288086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,8191,0.6700373490651449
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,1,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,7,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,16383,2.4876373608907065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,15,0.01791999985774358
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,63,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,31,0.01791999985774358
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,127,0.020992000897725422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,3,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,255,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,511,0.020138667275508244
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,2047,0.03669333209594091
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,4095,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,1023,0.024234667420387268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,8191,0.0747519979874293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,16383,0.10922666390736897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,3,0.012970666090647379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,7,0.012970666090647379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,15,0.013141332815090815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,31,0.012970666090647379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,1,0.012970666090647379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,127,0.012970666090647379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,255,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,63,0.013141332815090815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,511,0.020138667275508244
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,2047,0.040618665516376495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,1023,0.02867199977238973
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,4095,0.06775466601053874
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,16383,0.14028799533843994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,1,0.2582186659177144
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,7,0.2783573269844055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,3,0.2688000003496806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,15,0.3510613441467285
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,31,0.35140268007914227
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,63,0.3541333278020223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,8191,0.09693866968154907
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,127,0.42922667662302655
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,255,0.6830080350240072
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,511,1.1922773520151775
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,1,0.2590720057487488
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,3,0.26897066831588745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,1023,2.171050707499186
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,7,0.27989333868026733
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,31,0.35447466373443604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,63,0.3580586512883504
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,127,0.44287999471028644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,255,0.6995626290639242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,1,0.05495466788609823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,15,0.35549867153167725
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,511,1.1938133239746094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,1023,2.177365303039551
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,3,0.05495466788609823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,15,0.05461333195368449
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,7,0.05495466788609823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,31,0.05495466788609823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,63,0.05341866612434387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,255,0.07441066702206929
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,1023,0.1367039978504181
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,1,0.07594666878382365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,511,0.09574400385220845
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,3,0.07628799974918365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,7,0.07560533285140991
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,127,0.05734399954477946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,15,0.0766293356815974
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,31,0.07611733178297679
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,63,0.07714133461316426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,127,0.0865280032157898
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,511,0.1367039978504181
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,255,0.10376532872517903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,1023,0.21401600042978922
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,3,0.5331626733144125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,7,0.5536426703135172
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,31,0.7012693087259928
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,15,0.6985387007395426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,1,0.5138773520787557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,63,0.7121919790903727
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,255,1.3593600591023762
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,127,0.8690346876780192
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,1,0.5184853474299113
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,3,0.5346986850102743
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,7,0.5609813531239828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,15,0.7101439634958903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,127,0.8845653533935547
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,63,0.7253333727518717
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,1,0.0988159974416097
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,255,1.3893973032633464
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,3,0.10001066327095032
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,31,0.7190186977386475
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,7,0.10018133123715718
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,15,0.10069333513577779
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,31,0.10205866893132527
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,63,0.10291199882825215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,127,0.10820266604423523
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,255,0.130730668703715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,1,0.14148267110188803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,7,0.1418239971001943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,15,0.14284800489743552
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,31,0.14455466469128928
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,63,0.14353066682815552
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,127,0.15052800377209982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,3,0.14165332913398743
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,255,0.18875734011332193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,float16,1,1.0535253683725994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,float16,3,1.0922666390736897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,float16,7,1.132543961207072
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,float16,15,1.3991252581278484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,float16,31,1.4062933921813965
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,float16,63,1.426261266072591
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,float16,1,1.0574506918589275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,float16,3,1.0990933577219646
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,float16,7,1.141760031382243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,float16,127,1.7288533846537273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,float16,15,1.4216532707214355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,float16,31,1.430016040802002
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,float16,63,1.4434986114501953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,float16,127,1.7626454035441081
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,float16,1,0.19814399878184
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,float16,3,0.1976319948832194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,float16,7,0.19746132691701254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,float16,31,0.1971199909845988
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,float16,63,0.19524266322453818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,float16,15,0.1976319948832194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,float16,127,0.2001919945081075
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,float16,1,0.2795520027478536
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,float16,3,0.2800640066464742
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,float16,7,0.2795520027478536
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,float16,15,0.27989333868026733
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,1,0.022698665658632915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,float16,31,0.2788693308830261
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,float16,63,0.2775040070215861
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,3,0.023381332556406658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,7,0.023893333971500397
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,15,0.027647999425729115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,31,0.027818667391935985
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,float16,127,0.28381866216659546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,63,0.027989332874615986
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,255,0.05017599960168203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,127,0.03328000009059906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,511,0.08635733524958293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,1023,0.15547733505566916
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,2047,0.2986666758855184
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,4095,0.5642240047454834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,1,0.023039999107519787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,3,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,16383,4.542805353800456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,7,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,8191,1.2654933134714763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,15,0.027818667391935985
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,63,0.028330666323502857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,31,0.027818667391935985
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,127,0.03362133353948593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,255,0.051029334465662636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,511,0.08499200145403545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,1023,0.15411200126012167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,4095,0.5800960063934326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,8191,1.2404053211212158
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,2047,0.2903040051460266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,1,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,16383,4.576938629150391
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,3,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,15,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,7,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,31,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,255,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,511,0.0240639994541804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,1023,0.0339626669883728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,2047,0.048810665806134544
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,127,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,4095,0.08260266482830048
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,8191,0.1181013286113739
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,16383,0.17083734273910522
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,1,0.012970666090647379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,3,0.01331199953953425
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,7,0.013482666263977686
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,15,0.013141332815090815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,63,0.013141332815090815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,127,0.013141332815090815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,255,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,31,0.013823999712864557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,511,0.028330666323502857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,1023,0.04266666869322459
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,2047,0.06980266670385997
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,4095,0.10291199882825215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,8191,0.13619200388590494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,16383,0.20411733786265054
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,float16,1,2.104149341583252
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,float16,3,2.180095990498861
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,float16,15,2.7828906377156577
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,float16,7,2.252629280090332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,float16,31,2.799274762471517
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,float16,63,2.836480140686035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,float16,1,2.1056853930155435
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,float16,3,2.1853866577148438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,float16,7,2.27891206741333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,float16,15,2.8416001001993814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,float16,1,0.3816106716791789
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,float16,31,2.847231864929199
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,float16,3,0.38229334354400635
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,float16,63,2.8774401346842446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,float16,7,0.3804159959157308
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,float16,15,0.38229334354400635
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,float16,31,0.3831466833750407
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,float16,63,0.3763200044631958
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,float16,1,0.5464746554692587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,float16,3,0.5456213156382242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,float16,7,0.5454506476720175
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,float16,15,0.5454506476720175
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,float16,31,0.5437440077463785
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,float16,63,0.5423786640167236
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,float16,1,4.205055872599284
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,float16,3,4.357290585835774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,float16,7,4.496042569478353
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,float16,3,4.367018699645996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,float16,15,5.557589213053386
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,float16,1,4.212224006652832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,float16,7,4.527615865071614
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,float16,31,5.602303822835286
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,float16,1,0.7504213651021322
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,float16,31,5.701802571614583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,float16,15,5.671594619750977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,float16,3,0.7505919933319092
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,float16,15,0.7504213651021322
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,float16,7,0.7507627010345459
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,float16,31,0.7487146854400635
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,float16,1,1.0763946374257405
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,1,0.03703466554482778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,3,0.038058665891488395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,7,0.03976533313592275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,15,0.04898133377234141
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,31,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,63,0.04949333270390829
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,255,0.09062400460243225
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,127,0.059392000238100685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,float16,3,1.0752000013987224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,float16,7,1.075711965560913
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,float16,15,1.076053301493327
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,float16,31,1.072640021642049
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,511,0.16247466206550598
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,1023,0.2908160090446472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,2047,0.5630293289820353
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,4095,1.0808320045471191
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,1,0.03703466554482778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,8191,2.2872746785481772
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,3,0.03822933385769526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,7,0.03942399968703588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,15,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,31,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,63,0.04949333270390829
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,127,0.059562668204307556
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,255,0.09386666615804036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,511,0.15940266847610474
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,1023,0.2908160090446472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,2047,0.5533013343811035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,4095,1.070250670115153
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,1,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,3,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,8191,2.2821547190348306
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,7,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,15,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,31,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,63,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,127,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,255,0.02218666672706604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,511,0.034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,1023,0.051029334465662636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,2047,0.08516266942024231
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,4095,0.12322133779525757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,8191,0.16691199938456217
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,1,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,3,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,7,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,15,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,63,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,127,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,31,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,255,0.027477333943049114
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,511,0.042837331692377724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,1023,0.067071999112765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,2047,0.09762133161226909
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,4095,0.13312000036239624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,8191,0.20155733823776245
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,31,0.07167999943097432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,3,0.057002668579419456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,7,0.05836800237496694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,15,0.07133866846561432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,63,0.0721919983625412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,127,0.08703999718030293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,511,0.23586134115854898
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,1023,0.4278613328933716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,2047,0.8261973063151041
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,float16,1,0.05478399991989136
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,4095,1.5774720509847004
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,1,0.05478399991989136
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,float16,3,0.05717333157857259
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,float16,7,0.05819733440876007
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,float16,15,0.07167999943097432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,255,0.13619200388590494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,float16,31,0.07150933146476746
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,float16,63,0.07202133536338806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,float16,127,0.08686932921409607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,float16,1023,0.4333226680755615
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,float16,511,0.233130673567454
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,float16,2047,0.8209066390991211
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,float16,255,0.13567999998728433
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,float16,4095,1.5960747400919597
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,1,0.02218666672706604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,3,0.02218666672706604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,7,0.02218666672706604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,15,0.02218666672706604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,63,0.02218666672706604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,511,0.05222400029500326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,31,0.02218666672706604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,1023,0.07901866734027863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,127,0.02218666672706604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,2047,0.11776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,4095,0.15957333644231161
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,255,0.032255999743938446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,1,0.025941332181294758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,3,0.025770666698614757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,15,0.02611200014750163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,7,0.025770666698614757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,127,0.02611200014750163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,1023,0.0942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,511,0.0679253339767456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,63,0.025429333249727886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,2047,0.13448533415794373
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,31,0.02628266563018163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,4095,0.20411733786265054
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,1,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,3,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,7,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,15,0.011605333536863327
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,31,0.012629333883523941
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,511,0.03583999971548716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,255,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,1023,0.04164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,2047,0.06075733403364817
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,255,0.043178667624791466
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,127,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,4095,0.09113599856694539
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,float16,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,float16,3,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,16383,0.2645333409309387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,float16,7,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,8191,0.14882133404413858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,float16,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,float16,31,0.012970666090647379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,float16,255,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,float16,127,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,float16,511,0.03549866626660029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,float16,1023,0.040618665516376495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,float16,4095,0.08635733524958293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,float16,15,0.011776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,float16,2047,0.056832000613212585
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,float16,8191,0.14421332875887552
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,float16,16383,0.2571946581204732
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,7,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,255,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,1023,0.020138667275508244
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,511,0.03498666733503342
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,2047,0.023210667073726654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,127,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,4095,0.024234667420387268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,31,0.012629333883523941
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,8191,0.0314026673634847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,1,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,16383,0.046762665112813316
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,3,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,15,0.011605333536863327
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,31,0.012629333883523941
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,127,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,255,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,511,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,1023,0.0170666662355264
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,2047,0.020309332758188248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,8191,0.03532800078392029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,4095,0.028501334289709728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,1,0.010581333190202713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,16383,0.05273599922657013
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,3,0.010581333190202713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,7,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,31,0.013823999712864557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,255,0.017407999684413273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,63,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,511,0.025770666698614757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,1023,0.04232533276081085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,2047,0.07150933146476746
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,4095,0.1288533310095469
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,8191,0.23961599667867026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,16383,0.5302613178888956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,float16,3,0.010751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,127,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,float16,7,0.010751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,float16,1,0.010581333190202713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,float16,15,0.011776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,float16,31,0.013823999712864557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,float16,63,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,float16,127,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,float16,255,0.017407999684413273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,float16,1023,0.03942399968703588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,float16,4095,0.12288000186284383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,float16,8191,0.23125332593917847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,float16,16383,0.6312959988911947
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,float16,2047,0.0679253339767456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,1,0.010581333190202713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,3,0.010751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,7,0.010751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,float16,511,0.025600001215934753
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,15,0.011776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,31,0.013823999712864557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,63,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,255,0.0170666662355264
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,511,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,1023,0.020309332758188248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,4095,0.034645333886146545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,8191,0.044031997521718345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,16383,0.06946133573849995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,1,0.010581333190202713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,2047,0.024746666351954143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,7,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,15,0.011776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,31,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,63,0.013823999712864557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,255,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,2047,0.02867199977238973
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,1023,0.020138667275508244
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,8191,0.05751466751098633
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,4095,0.039594667653242745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,16383,0.08721066514650981
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,1,0.10154666503270467
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,3,0.10547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,511,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,7,0.10854400197664897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,15,0.13636266191800436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,31,0.1365333298842112
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,127,0.16520532965660095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,63,0.1378986636797587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,255,0.2619733413060506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,511,0.45721598466237384
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,float16,1,0.1013759970664978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,2047,1.589589277903239
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,float16,3,0.10547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,float16,7,0.10871466994285583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,float16,31,0.1367039978504181
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,float16,63,0.13738666971524557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,float16,15,0.1365333298842112
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,float16,127,0.16588800152142844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,float16,255,0.2624853253364563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,1023,0.839680035909017
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,float16,511,0.46250665187835693
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,float16,1023,0.8412160078684489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,1,0.032255999743938446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,float16,2047,1.5996586481730144
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,7,0.032255999743938446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,15,0.031231999397277832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,3,0.0314026673634847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,63,0.030378667016824085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,127,0.03089066594839096
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,255,0.05256533126036326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,511,0.08311466872692108
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,1023,0.11622400085131328
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,2047,0.16503467162450156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,3,0.03942399968703588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,7,0.03942399968703588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,1,0.039594667653242745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,15,0.03976533313592275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,31,0.039594667653242745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,31,0.0314026673634847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,63,0.03976533313592275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,127,0.03976533313592275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,255,0.057002668579419456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,511,0.07458133498827617
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,2047,0.1925119956334432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,1,0.017749333133300144
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,7,0.01791999985774358
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,15,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,1023,0.11161599556605022
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,31,0.023893333971500397
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,127,0.024405332903067272
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,63,0.023893333971500397
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,255,0.028501334289709728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,511,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,1023,0.07150933146476746
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,3,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,2047,0.1293653349081675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,4095,0.23808000485102335
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,8191,0.45772798856099445
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,float16,1,0.017749333133300144
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,16383,1.1064319610595703
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,float16,3,0.01791999985774358
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,float16,7,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,float16,63,0.024234667420387268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,float16,127,0.024234667420387268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,float16,511,0.043178667624791466
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,float16,255,0.02867199977238973
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,float16,1023,0.07048533360163371
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,float16,2047,0.12475732962290446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,float16,31,0.0240639994541804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,float16,4095,0.23483733336130777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,float16,8191,0.4613120158513387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,float16,15,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,float16,16383,1.1572906970977783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,7,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,3,0.01791999985774358
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,1,0.01757866640885671
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,15,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,31,0.024234667420387268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,63,0.024234667420387268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,255,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,511,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,1023,0.02372266600529353
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,2047,0.034815999368826546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,8191,0.07253333429495494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,16383,0.1063253382841746
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,1,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,3,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,4095,0.04693333307902018
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,7,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,127,0.024405332903067272
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,15,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,31,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,63,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,127,0.012117333710193634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,255,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,1023,0.027989332874615986
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,511,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,4095,0.06587733328342438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,8191,0.09557333588600159
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,16383,0.13858133554458618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,1,0.1955839991569519
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,3,0.20309333006540933
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,7,0.21128533283869425
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,2047,0.04130133241415024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,31,0.26692267258961994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,63,0.2688000003496806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,15,0.26743467648824054
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,127,0.32443734010060626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,255,0.5218986670176188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,511,0.9103360176086426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,1023,1.6424959500630696
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,float16,7,0.2121386726697286
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,float16,3,0.20377600193023682
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,float16,31,0.2677759925524394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,float16,63,0.26897066831588745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,float16,127,0.3300693432490031
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,float16,255,0.5333333412806193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,float16,1,0.19592533508936563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,float16,15,0.26828799645106
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,float16,511,0.9081172943115234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,1,0.051370665431022644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,3,0.051541333397229515
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,float16,1023,1.6621227264404297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,7,0.051882664362589516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,15,0.051541333397229515
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,63,0.050517335534095764
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,127,0.05222400029500326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,255,0.06843733290831248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,511,0.08994133273760478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,1023,0.13209600249926248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,1,0.07321600119272868
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,3,0.07338666419188182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,31,0.05171200136343638
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,7,0.07338666419188182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,15,0.07338666419188182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,31,0.07355733215808868
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,63,0.07372800012429555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,127,0.08157866696516673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,255,0.09915733337402344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,511,0.13192533453305563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,3,0.40004265308380127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,1,0.3863893349965413
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,7,0.4159146547317505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,15,0.5277013381322225
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,31,0.5292373498280843
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,63,0.5386240084966024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,127,0.6592853466669718
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,255,1.039189338684082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,96,2,128,0,1,float16,float16,3,0.4010666608810425
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,96,2,128,0,1,float16,float16,1,0.3867306709289551
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,96,2,128,0,1,float16,float16,15,0.5319679975509644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,96,2,128,0,1,float16,float16,7,0.4184746742248535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,1023,0.20753065745035806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,96,2,128,0,1,float16,float16,31,0.5362346569697062
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,96,2,128,0,1,float16,float16,63,0.5476693312327067
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,96,2,128,0,1,float16,float16,127,0.6748159726460775
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,96,2,128,0,1,float16,float16,255,1.0525013605753581
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,3,0.09250133236249287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,1,0.09335466225941975
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,7,0.09233066439628601
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,31,0.09386666615804036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,63,0.09471999605496724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,127,0.10188800096511841
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,1,0.13499733805656433
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,255,0.12151466806729634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,15,0.09403733412424724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,7,0.1360213359196981
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,31,0.1384106675783793
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,15,0.1365333298842112
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,63,0.1389226714769999
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,127,0.1460906664530436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,255,0.18295466899871826
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,float16,3,0.8209066390991211
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,float16,7,0.8541866938273112
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,float16,15,1.061205307642619
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,3,0.1353386640548706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,float16,1,0.7942826747894287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,float16,31,1.0671786467234294
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,float16,63,1.0821973482767742
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,float16,127,1.3081599871317546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,96,2,128,0,1,float16,float16,1,0.8004266421000162
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,96,2,128,0,1,float16,float16,3,0.8328533172607422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,96,2,128,0,1,float16,float16,7,0.8674986362457275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,96,2,128,0,1,float16,float16,15,1.068714698155721
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,96,2,128,0,1,float16,float16,63,1.0895360310872395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,96,2,128,0,1,float16,float16,127,1.3414400418599446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,float16,1,0.1843199928601583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,float16,3,0.18397865692774454
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,float16,7,0.1848319967587789
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,float16,15,0.18722132841746011
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,96,2,128,0,1,float16,float16,31,1.0810026327768962
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,float16,63,0.1838080088297526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,float16,31,0.1879040002822876
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,float16,127,0.1904639999071757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,float16,1,0.26948267221450806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,float16,3,0.27460267146428424
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,float16,7,0.2691413362820943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,float16,15,0.26845866441726685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,1,0.017237332959969837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,3,0.017749333133300144
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,7,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,float16,63,0.2677759925524394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,31,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,63,0.023039999107519787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,127,0.027306665976842243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,float16,127,0.2727253238360087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,15,0.023039999107519787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,255,0.040789333482583366
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,511,0.0682666649421056
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,float16,31,0.2686293323834737
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,1023,0.12270933389663696
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,2047,0.23057067394256592
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,4095,0.4432213306427002
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,float16,1,0.017237332959969837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,8191,0.9855999946594238
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,float16,3,0.01791999985774358
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,16383,3.1793492635091147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,float16,15,0.022698665658632915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,float16,31,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,float16,63,0.023039999107519787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,float16,127,0.027306665976842243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,float16,511,0.0682666649421056
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,float16,255,0.04130133241415024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,float16,7,0.018602666755517323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,float16,2047,0.23125332593917847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,float16,1023,0.1225386659304301
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,float16,4095,0.45499734083811444
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,float16,8191,0.9871359666188558
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,1,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,15,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,float16,16383,3.359231948852539
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,3,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,31,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,63,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,255,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,7,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,511,0.023210667073726654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,1023,0.03259733319282532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,4095,0.08021333316961925
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,2047,0.048469334840774536
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,8191,0.1153706709543864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,1,0.012970666090647379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,16383,0.16708266735076904
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,7,0.012970666090647379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,15,0.012970666090647379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,31,0.012970666090647379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,63,0.013141332815090815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,127,0.013141332815090815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,255,0.018432000031073887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,1023,0.04164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,511,0.028160000840822857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,2047,0.06877866884072621
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,4095,0.10018133123715718
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,3,0.012800000607967377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,16383,0.20189867417017618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,8191,0.13363200426101685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,float16,1,1.583445390065511
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,float16,3,1.64027738571167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,float16,7,1.7025705973307292
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,float16,31,2.1256532669067383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,96,2,128,0,1,float16,float16,1,1.605631987253825
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,float16,15,2.1142187118530273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,96,2,128,0,1,float16,float16,3,1.6626346906026204
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,96,2,128,0,1,float16,float16,7,1.723050594329834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,96,2,128,0,1,float16,float16,31,2.144597371419271
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,float16,63,2.1580799420674643
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,96,2,128,0,1,float16,float16,15,2.13265069325765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,96,2,128,0,1,float16,float16,63,2.164735953013102
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,float16,1,0.36881065368652344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,float16,7,0.36881065368652344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,float16,3,0.3671040137608846
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,float16,15,0.3671040137608846
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,float16,31,0.366592009862264
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,float16,1,0.5275306701660156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,float16,3,0.527018666267395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,float16,7,0.5256533225377401
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,float16,15,0.5256533225377401
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,float16,31,0.5241173505783081
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,float16,63,0.35891199111938477
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,float16,63,0.5249706506729126
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,float16,1,3.1639893849690757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,float16,3,3.277141253153483
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,float16,7,3.3978026707967124
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,float16,15,4.22877852121989
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,float16,31,4.253525416056315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,96,2,128,0,1,float16,float16,1,3.1994880040486655
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,96,2,128,0,1,float16,float16,7,3.430570602416992
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,96,2,128,0,1,float16,float16,3,3.3160533905029297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,96,2,128,0,1,float16,float16,15,4.2639360427856445
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,96,2,128,0,1,float16,float16,31,4.275882720947266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,float16,1,0.7217493057250977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,float16,3,0.7203839619954427
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,float16,15,0.7195306619008383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,float16,31,0.7181653181711832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,1,0.029696000119050343
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,float16,7,0.7202133337656657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,float16,3,1.0412373542785645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,3,0.03054933249950409
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,7,0.0315733328461647
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,float16,7,1.0410666465759277
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,float16,15,1.0385066668192546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,float16,1,1.0426026980082195
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,float16,31,1.0385066668192546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,31,0.038912000755469
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,63,0.03942399968703588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,15,0.03942399968703588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,255,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,127,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,511,0.12441600362459819
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,2047,0.4254719813664754
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,4095,0.8340480327606201
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,1023,0.22425599892934164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,8191,1.7544533411661785
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,float16,3,0.03054933249950409
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,float16,1,0.029866665601730347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,float16,7,0.0315733328461647
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,float16,15,0.03908266623814901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,float16,31,0.03908266623814901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,float16,255,0.07287466526031494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,float16,63,0.03925333420435587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,float16,511,0.12407466769218445
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,float16,127,0.04778666794300079
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,float16,2047,0.43161598841349286
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,float16,1023,0.22254933913548788
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,float16,4095,0.8343893686930338
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,float16,8191,1.7745919227600098
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,1,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,7,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,3,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,31,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,127,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,15,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,255,0.021333334346612293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,511,0.03345066557327906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,1023,0.04966400067011515
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,2047,0.08362666765848796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,4095,0.1230506698290507
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,8191,0.1634986698627472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,7,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,1,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,15,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,31,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,63,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,3,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,127,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,255,0.02679466704527537
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,511,0.041984001795450844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,2047,0.09523199995358785
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,4095,0.13192533453305563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,8191,0.20121600230534872
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,1023,0.06621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,1,0.03703466554482778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,3,0.03822933385769526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,31,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,15,0.051541333397229515
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,7,0.03925333420435587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,255,0.09301333626111348
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,511,0.15923200050989786
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,1023,0.2926933368047078
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,2047,0.5602986812591553
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,4095,1.0786133607228596
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,8191,2.306730588277181
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,63,0.04949333270390829
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,127,0.05973333120346069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,7,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,31,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,1,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,3,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,15,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,511,0.034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,1023,0.05085866649945577
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,4095,0.1237333317597707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,1,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,2047,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,8191,0.16742400328318277
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,3,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,7,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,127,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,15,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,31,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,63,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,255,0.02218666672706604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,127,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,255,0.027306665976842243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,511,0.043178667624791466
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,2047,0.09898666540781657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,8191,0.2039466698964437
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,4095,0.13294933239618936
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,1023,0.06758399804433186
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,3,0.025258667767047882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,15,0.025258667767047882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,7,0.025429333249727886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,63,0.025258667767047882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,511,0.06673066814740498
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,1,0.025770666698614757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,2047,0.13346133629480997
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,31,0.025258667767047882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,1023,0.09267200032869975
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,127,0.025258667767047882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,4095,0.20480000972747803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,8191,0.36317865053812665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,31,0.012458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,255,0.04215466479460398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,63,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,255,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,511,0.03379199902216593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,1023,0.03942399968703588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,2047,0.05495466788609823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,8191,0.14097066720326742
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,1,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,16383,0.2558293342590332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,32767,0.48264535268147785
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,4095,0.08533333738644917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,15,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,255,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,31,0.012629333883523941
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,511,0.03362133353948593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,1023,0.039936001102129616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,4095,0.023381332556406658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,8191,0.030378667016824085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,16383,0.03328000009059906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,32767,0.04249600072701772
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,15,0.011605333536863327
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,7,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,1,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,31,0.012458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,255,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,2047,0.022357332209746044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,511,0.03345066557327906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,1023,0.0170666662355264
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,4095,0.020309332758188248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,8191,0.02611200014750163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,2047,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,16383,0.03925333420435587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,3,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,32767,0.053930665055910744
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,7,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,15,0.011776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,31,0.012458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,255,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,511,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,2047,0.019968000551064808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,1023,0.016895999511082966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,4095,0.02935466667016347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,8191,0.03618133316437403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,1,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,16383,0.05256533126036326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,32767,0.08311466872692108
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,3,0.01877333347996076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,7,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,31,0.021674667795499165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,15,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,1,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,127,0.025941332181294758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,255,0.0264533335963885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,511,0.030378667016824085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,63,0.02611200014750163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,1023,0.04539733131726583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,2047,0.07458133498827617
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,4095,0.12544000148773193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,16383,0.4416853189468384
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,1,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,8191,0.23125332593917847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,32767,0.7516160011291504
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,15,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,7,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,31,0.021674667795499165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,63,0.025941332181294758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,3,0.01877333347996076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,127,0.025941332181294758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,511,0.03089066594839096
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,1023,0.020138667275508244
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,255,0.02679466704527537
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,2047,0.023381332556406658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,4095,0.025429333249727886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,8191,0.03276800115903219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,32767,0.06980266670385997
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,1,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,16383,0.04949333270390829
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,3,0.01877333347996076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,31,0.021674667795499165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,15,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,63,0.026965332527955372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,7,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,127,0.02611200014750163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,255,0.0264533335963885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,1023,0.0170666662355264
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,2047,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,511,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,4095,0.030378667016824085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,8191,0.036864000062147774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,32767,0.08567466338475545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,1,0.01877333347996076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,3,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,7,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,16383,0.05922133227189382
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,31,0.021333334346612293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,63,0.025770666698614757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,15,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,127,0.025941332181294758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,255,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,511,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,1023,0.019968000551064808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,4095,0.03908266623814901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,8191,0.057002668579419456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,2047,0.0290133332212766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,32767,0.13380266229311624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,16383,0.08635733524958293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,1,0.06963199873765309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,3,0.0721919983625412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,7,0.07458133498827617
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,15,0.09284266829490662
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,31,0.09284266829490662
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,127,0.1129813293615977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,1023,0.570197343826294
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,511,0.30856533845265705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,63,0.09386666615804036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,255,0.17800533771514893
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,2047,1.0767359733581543
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,1,0.023039999107519787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,4095,2.0802559852600098
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,3,0.023210667073726654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,7,0.023039999107519787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,15,0.023210667073726654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,127,0.023210667073726654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,255,0.03293866664171219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,63,0.022698665658632915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,31,0.023210667073726654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,511,0.052906667192777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,2047,0.12032000223795573
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,1023,0.08157866696516673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,1,0.02628266563018163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,4095,0.16366933782895407
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,3,0.02628266563018163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,7,0.02628266563018163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,31,0.0264533335963885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,63,0.02628266563018163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,127,0.02628266563018163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,255,0.04351999859015147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,15,0.0264533335963885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,511,0.06963199873765309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,1023,0.09642666578292847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,1,0.03857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,4095,0.2065066695213318
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,2047,0.1367039978504181
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,3,0.03857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,31,0.03857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,63,0.03857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,15,0.03874133278926214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,127,0.03874133278926214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,255,0.05444266895453135
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,2047,0.1919999917348226
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,511,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,4095,0.3421866496404012
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,1023,0.11059199770291646
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,7,0.03857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,3,0.01757866640885671
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,1,0.017237332959969837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,7,0.01757866640885671
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,15,0.017749333133300144
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,63,0.020309332758188248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,31,0.019968000551064808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,127,0.020821332931518555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,511,0.034815999368826546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,2047,0.10018133123715718
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,4095,0.18193066120147705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,8191,0.34781865278879803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,1023,0.057855998476346336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,16383,0.8920746644337972
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,1,0.0170666662355264
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,255,0.024234667420387268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,32767,1.6448853810628254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,3,0.01757866640885671
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,7,0.01757866640885671
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,15,0.017749333133300144
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,127,0.02065066620707512
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,63,0.020309332758188248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,31,0.020309332758188248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,511,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,2047,0.02628266563018163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,4095,0.037205333511034645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,1023,0.020138667275508244
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,8191,0.04607999821503957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,16383,0.07372800012429555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,255,0.024234667420387268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,1,0.017407999684413273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,3,0.017749333133300144
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,7,0.017749333133300144
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,15,0.01757866640885671
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,32767,0.106495996316274
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,31,0.019968000551064808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,63,0.020138667275508244
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,511,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,1023,0.020138667275508244
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,255,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,2047,0.030207999050617218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,8191,0.06178133189678192
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,4095,0.04147200038035711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,16383,0.09096533060073853
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,127,0.020821332931518555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,1,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,15,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,32767,0.17032533884048462
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,3,0.012117333710193634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,7,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,63,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,31,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,255,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,511,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,1023,0.027306665976842243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,2047,0.05597866574923197
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,4095,0.06400000055631001
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,8191,0.09574400385220845
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,127,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,1,0.13260799646377563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,32767,0.2053119937578837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,3,0.13738666971524557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,15,0.1802240014076233
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,7,0.1443839967250824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,16383,0.13994666934013367
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,31,0.1802240014076233
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,127,0.2193066676457723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,511,0.6142293214797974
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,1023,1.1124053001403809
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,2047,2.1009066899617515
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,255,0.34918399651845294
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,1,0.034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,7,0.034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,63,0.18175999323527017
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,15,0.034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,31,0.034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,3,0.034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,63,0.03379199902216593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,511,0.08755200107892354
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,127,0.03379199902216593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,255,0.057855998476346336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,2047,0.1699840029080709
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,1,0.04266666869322459
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,7,0.04266666869322459
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,3,0.042837331692377724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,15,0.04249600072701772
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,1023,0.11997866630554199
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,31,0.04266666869322459
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,127,0.04266666869322459
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,255,0.06058666606744131
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,63,0.04266666869322459
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,511,0.07850666840871175
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,2047,0.2001919945081075
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,1023,0.11827199657758077
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,3,0.07321600119272868
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,1,0.07338666419188182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,7,0.07321600119272868
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,15,0.07338666419188182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,127,0.08089600006739299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,31,0.07355733215808868
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,63,0.07338666419188182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,255,0.09847467144330342
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,511,0.130730668703715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,2047,0.3604480028152466
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,1,0.25941334168116253
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,3,0.26948267221450806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,1023,0.20514132579167685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,15,0.3561813433965047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,63,0.3580586512883504
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,31,0.3548159996668498
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,7,0.2815999984741211
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,127,0.4442453384399414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,255,0.6993920008341471
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,511,1.2142933209737141
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,3,0.05580799778302511
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,7,0.0554666668176651
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,1,0.05580799778302511
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,15,0.05597866574923197
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,63,0.05358933409055074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,31,0.05529599885145823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,127,0.0580266664425532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,255,0.07577600081761678
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,511,0.09676800171534221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,1,0.07645866771539052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,15,0.07645866771539052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,7,0.07645866771539052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,3,0.0766293356815974
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,63,0.07850666840871175
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,127,0.08260266482830048
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,31,0.0769706666469574
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,255,0.10376532872517903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,511,0.1384106675783793
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,1,0.13158399860064188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,15,0.13260799646377563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,31,0.13414399822553
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,3,0.13175466656684875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,7,0.14984533190727234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,63,0.1360213359196981
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,127,0.14062933127085367
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,255,0.1786880095799764
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,511,0.2404693365097046
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,1,0.5157546599706014
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,3,0.5381120045979818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,7,0.5626879930496216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,15,0.7087786992390951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,31,0.7275520165761312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,255,1.384618600209554
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,127,0.8832000096638998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,63,0.7280639807383219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,3,0.10069333513577779
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,1,0.09949866930643718
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,7,0.10001066327095032
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,31,0.10376532872517903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,15,0.10188800096511841
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,63,0.10222933689753215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,127,0.10820266604423523
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,255,0.130730668703715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,1,0.1418239971001943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,3,0.14199466506640115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,7,0.1423360009988149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,63,0.14404267072677612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,15,0.14387200276056925
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,127,0.15103999773661295
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,1,0.2585600018501282
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,31,0.14523733655611673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,255,0.18995199600855509
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,3,0.2583893338839213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,7,0.258730669816335
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,15,0.25924267371495563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,3,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,31,0.25924267371495563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,63,0.2595840096473694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,1,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,127,0.26470400889714557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,7,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,255,0.3415040175120036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,15,0.017749333133300144
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,31,0.017749333133300144
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,127,0.020821332931518555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,63,0.018432000031073887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,511,0.04795733094215393
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,1023,0.09437867005666097
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,255,0.029866665601730347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,4095,0.2988373239835103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,8191,0.668842633565267
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,2047,0.15615999698638916
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,1,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,3,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,16383,2.4693759282430015
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,7,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,32767,4.9966080983479815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,15,0.017749333133300144
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,31,0.01791999985774358
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,127,0.020821332931518555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,511,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,255,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,63,0.018432000031073887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,1023,0.023210667073726654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,4095,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,2047,0.03618133316437403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,8191,0.07509333391984303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,16383,0.10939733187357585
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,1,0.011776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,32767,0.16759467124938965
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,7,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,3,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,15,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,63,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,127,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,31,0.012117333710193634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,255,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,511,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,1023,0.027989332874615986
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,4095,0.0682666649421056
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,2047,0.04113066693147024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,16383,0.14079999923706055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,8191,0.09693866968154907
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,1,0.012800000607967377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,3,0.012970666090647379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,32767,0.20855466524759927
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,7,0.012970666090647379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,15,0.012458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,63,0.012117333710193634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,127,0.012970666090647379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,255,0.018432000031073887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,31,0.012800000607967377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,511,0.027306665976842243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,1023,0.04113066693147024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,2047,0.06724266707897186
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,4095,0.09983999530474345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,8191,0.13516799608866373
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,16383,0.20002132654190063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,32767,0.3546453317006429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,float16,3,1.098581314086914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,float16,7,1.1438079675038655
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,float16,15,1.4293333689371746
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,float16,1,1.0615466435750325
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,float16,31,1.427626609802246
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,float16,63,1.4458880424499512
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,float16,1,0.19814399878184
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,float16,127,1.7553067207336426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,float16,3,0.19729065895080566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,float16,15,0.19814399878184
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,float16,7,0.19814399878184
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,float16,63,0.1955839991569519
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,float16,127,0.20053333044052124
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,float16,31,0.19746132691701254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,float16,1,0.2797226707140605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,float16,3,0.2795520027478536
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,float16,7,0.2786986629168193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,float16,31,0.2786986629168193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,float16,15,0.27852799495061237
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,float16,63,0.276309331258138
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,float16,127,0.28330665826797485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,float16,3,0.5102933247884115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,float16,7,0.5089279810587565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,float16,1,0.5111466646194458
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,float16,15,0.5089279810587565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,float16,31,0.5090986490249634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,float16,127,0.5131946802139282
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,float16,63,0.5092693169911703
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,float16,1,2.118314743041992
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,float16,3,2.1975040435791016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,float16,7,2.2743040720621743
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,float16,31,2.8516693115234375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,float16,15,2.8388694127400718
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,float16,1,0.38519465923309326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,float16,3,0.3850239912668864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,float16,63,2.8929707209269204
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,float16,7,0.3848533233006795
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,float16,15,0.3845119873682658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,float16,31,0.38570666313171387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,float16,63,0.37614933649698895
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,float16,3,0.5439146757125854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,float16,7,0.5459626515706381
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,float16,15,0.5452800194422404
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,float16,31,0.5427199999491373
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,float16,1,0.5457919836044312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,float16,63,0.5406719843546549
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,float16,1,1.0091520150502522
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,float16,3,1.008128007253011
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,float16,7,1.0079572995503743
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,3,0.023210667073726654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,7,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,float16,31,1.0072746276855469
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,float16,15,1.008128007253011
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,15,0.027647999425729115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,float16,63,1.0074453353881836
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,31,0.027818667391935985
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,63,0.028501334289709728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,127,0.03362133353948593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,255,0.05085866649945577
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,511,0.08516266942024231
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,1,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,1023,0.15786666671435037
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,2047,0.28945066531499225
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,4095,0.5777066548665365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,8191,1.2392106850941975
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,1,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,3,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,16383,4.546559969584147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,31,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,15,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,63,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,255,0.01877333347996076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,511,0.023210667073726654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,1023,0.03328000009059906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,7,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,2047,0.04949333270390829
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,4095,0.08311466872692108
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,8191,0.11878400047620137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,1,0.013141332815090815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,16383,0.17169066270192465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,3,0.013141332815090815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,7,0.012970666090647379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,31,0.012970666090647379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,63,0.012800000607967377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,127,0.012800000607967377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,255,0.018602666755517323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,511,0.027989332874615986
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,2047,0.07031466563542683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,1023,0.04181333382924398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,4095,0.10274133086204529
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,8191,0.1378986636797587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,15,0.012970666090647379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,16383,0.20565332969029745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,1,0.018602666755517323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,3,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,7,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,15,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,31,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,63,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,255,0.0264533335963885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,511,0.04164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,1023,0.06417066852251689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,2047,0.09471999605496724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,8191,0.20155733823776245
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,4095,0.1302186648050944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,16383,0.3604480028152466
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,127,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,15,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,31,0.013823999712864557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,63,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,7,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,255,0.018432000031073887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,1,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,1023,0.03345066557327906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,511,0.023381332556406658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,2047,0.04949333270390829
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,4095,0.08294400076071422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,8191,0.11946666240692139
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,16383,0.17203199863433838
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,3,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,3,0.013141332815090815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,15,0.012970666090647379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,31,0.012970666090647379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,7,0.012970666090647379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,63,0.012970666090647379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,127,0.012970666090647379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,511,0.027647999425729115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,1023,0.04181333382924398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,2047,0.07150933146476746
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,1,0.012970666090647379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,4095,0.10410666465759277
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,8191,0.13772799571355185
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,16383,0.2065066695213318
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,1,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,3,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,7,0.01877333347996076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,63,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,31,0.029696000119050343
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,127,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,255,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,255,0.02628266563018163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,511,0.04147200038035711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,1023,0.06502399841944377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,2047,0.0962559978167216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,4095,0.13209600249926248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,15,0.018432000031073887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,8191,0.202239990234375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,16383,0.36215468247731525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,1,0.025087999800841015
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,3,0.025087999800841015
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,7,0.02491733431816101
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,15,0.025258667767047882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,31,0.025087999800841015
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,127,0.025087999800841015
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,511,0.06553600231806438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,63,0.025087999800841015
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,1023,0.09113599856694539
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,2047,0.13004799683888754
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,4095,0.2027519941329956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,8191,0.35788798332214355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,16383,0.6763520240783691
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,7,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,15,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,31,0.012458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,255,0.04113066693147024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,255,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,63,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,511,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,1023,0.021674667795499165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,4095,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,2047,0.03925333420435587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,16383,0.03259733319282532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,32767,0.04147200038035711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,65535,0.051029334465662636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,8191,0.030037333567937214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,63,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,31,0.012458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,255,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,511,0.019968000551064808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,1023,0.02184533327817917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,8191,0.025087999800841015
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,16383,0.027306665976842243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,4095,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,32767,0.03498666733503342
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,65535,0.08021333316961925
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,2047,0.018432000031073887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,15,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,63,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,255,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,31,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,1023,0.016895999511082966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,2047,0.01877333347996076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,8191,0.025941332181294758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,4095,0.020309332758188248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,511,0.019968000551064808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,16383,0.03857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,32767,0.051029334465662636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,65535,0.08499200145403545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,7,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,63,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,15,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,255,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,511,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,2047,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,31,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,1023,0.0170666662355264
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,4095,0.0288426677385966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,16383,0.05171200136343638
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,32767,0.08123733103275299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,8191,0.034815999368826546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,65535,0.1262933313846588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,31,0.012458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,7,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,63,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,255,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,511,0.03362133353948593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,1023,0.03925333420435587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,2047,0.023210667073726654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,8191,0.030720000465710957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,4095,0.024746666351954143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,16383,0.03362133353948593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,3,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,65535,0.07987200220425923
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,32767,0.04334933559099833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,15,0.011605333536863327
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,63,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,31,0.012458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,255,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,511,0.03362133353948593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,1023,0.0170666662355264
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,2047,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,4095,0.021162666380405426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,16383,0.039936001102129616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,65535,0.0885759989420573
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,32767,0.05461333195368449
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,8191,0.027306665976842243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,31,0.012458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,63,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,255,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,1023,0.0170666662355264
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,2047,0.020821332931518555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,511,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,8191,0.03618133316437403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,16383,0.05529599885145823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,32767,0.0846506655216217
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,4095,0.02935466667016347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,65535,0.12851199507713318
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,7,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,63,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,31,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,511,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,1023,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,2047,0.028160000840822857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,8191,0.05563733478387197
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,16383,0.08499200145403545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,255,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,32767,0.13107200463612875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,4095,0.037717332442601524
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,65535,0.21435733636220297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,1,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,3,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,7,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,15,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,255,0.022357332209746044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,1023,0.05085866649945577
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,511,0.034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,2047,0.08618666728337605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,127,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,4095,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,31,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,8191,0.16844799121220908
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,3,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,1,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,7,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,15,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,63,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,31,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,255,0.027477333943049114
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,511,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,1023,0.06724266707897186
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,2047,0.10018133123715718
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,4095,0.13516799608866373
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,8191,0.2053119937578837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,127,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,3,0.025258667767047882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,7,0.025258667767047882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,1,0.025258667767047882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,15,0.025258667767047882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,31,0.025258667767047882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,63,0.025258667767047882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,127,0.025600001215934753
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,511,0.06758399804433186
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,1023,0.09386666615804036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,1,0.03839999934037527
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,8191,0.36403199036916095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,3,0.03822933385769526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,4095,0.2039466698964437
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,15,0.03857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,7,0.03839999934037527
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,2047,0.1353386640548706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,255,0.04215466479460398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,31,0.03788800040880839
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,63,0.03839999934037527
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,127,0.03839999934037527
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,511,0.07133866846561432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,1023,0.10871466994285583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,255,0.053247998158137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,4095,0.33826132615407306
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,1,0.01877333347996076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,3,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,2047,0.1889280080795288
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,8191,0.6393173138300577
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,7,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,15,0.019968000551064808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,31,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,127,0.02611200014750163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,255,0.0264533335963885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,1023,0.019968000551064808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,2047,0.024405332903067272
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,4095,0.02628266563018163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,63,0.02611200014750163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,511,0.03054933249950409
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,8191,0.03379199902216593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,16383,0.05017599960168203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,1,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,3,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,32767,0.07031466563542683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,65535,0.10905599594116211
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,7,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,63,0.025941332181294758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,127,0.02611200014750163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,31,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,255,0.0266239990790685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,1023,0.0170666662355264
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,511,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,15,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,2047,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,8191,0.03874133278926214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,4095,0.0314026673634847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,32767,0.08755200107892354
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,1,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,65535,0.1353386640548706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,16383,0.05905066430568695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,3,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,15,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,31,0.021333334346612293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,63,0.025941332181294758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,127,0.025941332181294758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,255,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,7,0.01877333347996076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,511,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,2047,0.03054933249950409
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,4095,0.04027733455101649
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,1023,0.019968000551064808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,8191,0.059392000238100685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,32767,0.1365333298842112
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,16383,0.08925867080688477
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,3,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,1,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,7,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,65535,0.21845332781473795
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,63,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,31,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,127,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,15,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,255,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,1023,0.027136000494162243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,4095,0.06297599772612254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,511,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,8191,0.09403733412424724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,16383,0.1365333298842112
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,2047,0.03976533313592275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,65535,0.3633493185043335
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,1,0.023210667073726654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,32767,0.2027519941329956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,3,0.023210667073726654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,15,0.023210667073726654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,7,0.023210667073726654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,31,0.023210667073726654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,63,0.023039999107519787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,127,0.023210667073726654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,255,0.03310933212439219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,1023,0.0820906658967336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,2047,0.12014933427174886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,1,0.0264533335963885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,511,0.05358933409055074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,4095,0.1646933356920878
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,7,0.0264533335963885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,15,0.02628266563018163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,31,0.02628266563018163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,3,0.02628266563018163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,63,0.02611200014750163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,127,0.02628266563018163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,1023,0.09693866968154907
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,511,0.07031466563542683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,255,0.044031997521718345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,2047,0.13772799571355185
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,4095,0.20923733711242676
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,3,0.03857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,1,0.03857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,7,0.03857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,15,0.03857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,127,0.03857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,255,0.0554666668176651
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,31,0.03857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,511,0.0730453332265218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,63,0.03857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,2047,0.19234132766723633
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,4095,0.339626669883728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,1023,0.1129813293615977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,1,0.06929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,3,0.06929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,15,0.06929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,7,0.06929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,127,0.07389866809050243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,63,0.06929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,511,0.12595199545224509
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,31,0.06929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,1023,0.1986560026804606
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,2047,0.35703468322753906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,255,0.09386666615804036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,1,0.03379199902216593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,4095,0.6533120075861613
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,3,0.034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,7,0.034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,31,0.034304000437259674
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,63,0.03293866664171219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,15,0.03379199902216593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,127,0.03328000009059906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,255,0.05580799778302511
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,511,0.08721066514650981
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,1,0.04164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,3,0.04164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,1023,0.11929600437482198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,7,0.04164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,15,0.04164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,255,0.05973333120346069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,511,0.07867733140786488
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,31,0.04164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,1023,0.1181013286113739
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,127,0.04164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,1,0.0721919983625412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,63,0.04164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,7,0.07236266632874806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,15,0.07236266632874806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,63,0.07236266632874806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,255,0.09727999567985535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,127,0.08004266520341237
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,3,0.07236266632874806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,31,0.07236266632874806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,1,0.1276586651802063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,1023,0.20258132616678873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,3,0.12782933314641318
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,7,0.12748799721399942
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,15,0.1276586651802063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,63,0.12919466694196066
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,31,0.1276586651802063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,127,0.1353386640548706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,511,0.13107200463612875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,255,0.17356799046198526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,511,0.2326186696688334
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,1023,0.3671040137608846
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,1,0.05529599885145823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,3,0.07116800049940745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,15,0.0554666668176651
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,63,0.054272000988324486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,7,0.05529599885145823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,127,0.0580266664425532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,255,0.07577600081761678
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,511,0.09693866968154907
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,3,0.07645866771539052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,7,0.07628799974918365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,31,0.05597866574923197
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,1,0.0766293356815974
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,31,0.0766293356815974
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,63,0.07867733140786488
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,15,0.07645866771539052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,255,0.10513066252072652
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,127,0.08550399541854858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,511,0.1418239971001943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,3,0.13175466656684875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,7,0.13192533453305563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,15,0.13260799646377563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,1,0.13175466656684875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,63,0.1360213359196981
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,31,0.13431466619173685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,255,0.1802240014076233
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,127,0.1423360009988149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,1,0.24832000335057577
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,3,0.24849067131678262
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,511,0.24251733223597208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,15,0.2505386670430501
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,63,0.25173334280649823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,127,0.25548799832661945
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,7,0.2491733431816101
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,31,0.25088000297546387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,511,0.4476586580276489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,255,0.3309226632118225
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,1,0.0170666662355264
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,3,0.017237332959969837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,7,0.01757866640885671
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,15,0.017749333133300144
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,127,0.020821332931518555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,1023,0.020138667275508244
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,63,0.02065066620707512
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,511,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,2047,0.025600001215934753
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,31,0.020309332758188248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,4095,0.03669333209594091
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,8191,0.04693333307902018
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,16383,0.07338666419188182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,3,0.01757866640885671
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,65535,0.16059733430544534
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,32767,0.10700800021489461
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,1,0.017237332959969837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,31,0.020138667275508244
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,63,0.020309332758188248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,15,0.017749333133300144
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,7,0.017749333133300144
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,127,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,255,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,2047,0.030207999050617218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,4095,0.04147200038035711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,511,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,8191,0.061610668897628784
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,1023,0.020138667275508244
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,32767,0.13858133554458618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,1,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,3,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,16383,0.09079466263453166
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,65535,0.222378671169281
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,15,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,63,0.012117333710193634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,7,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,31,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,127,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,255,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,1023,0.027306665976842243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,4095,0.06621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,2047,0.040448000033696495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,511,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,32767,0.2063360015551249
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,8191,0.0962559978167216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,65535,0.3689813216527303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,16383,0.13943466544151306
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,3,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,1,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,7,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,31,0.012458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,127,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,511,0.027306665976842243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,63,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,15,0.012629333883523941
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,255,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,1023,0.040618665516376495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,2047,0.06673066814740498
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,4095,0.09796266754468282
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,8191,0.13294933239618936
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,16383,0.20036266247431436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,3,0.09915733337402344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,7,0.10001066327095032
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,15,0.1013759970664978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,65535,0.6478506724039713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,32767,0.3495253324508667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,31,0.10274133086204529
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,63,0.10359467069307964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,1,0.10001066327095032
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,1,0.14114133516947427
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,3,0.14131200313568115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,127,0.1083733340104421
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,15,0.14353066682815552
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,255,0.13260799646377563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,31,0.14506666858990988
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,7,0.14114133516947427
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,63,0.14404267072677612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,127,0.1704960068066915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,3,0.2583893338839213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,1,0.25941334168116253
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,7,0.2604373296101888
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,255,0.19268266359965006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,15,0.25924267371495563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,63,0.2606079975763957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,127,0.2657279968261719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,31,0.25941334168116253
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,3,0.49373865127563477
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,255,0.3433813254038493
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,1,0.49373865127563477
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,7,0.4939093192418416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,15,0.49322664737701416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,63,0.4928853511810303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,31,0.4930560191472371
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,255,0.6439253489176432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,float16,1,0.19780266284942627
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,float16,3,0.19746132691701254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,float16,7,0.19746132691701254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,float16,15,0.1967786749204
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,127,0.5003946622212728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,float16,31,0.19643733898798624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,float16,63,0.19592533508936563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,float16,1,0.27801599105199176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,float16,7,0.27852799495061237
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,float16,3,0.27801599105199176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,float16,15,0.2764799992243449
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,float16,127,0.1996799906094869
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,float16,31,0.27613866329193115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,float16,63,0.2757973273595174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,float16,1,0.509610652923584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,float16,3,0.5084160168965658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,float16,127,0.2815999984741211
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,float16,15,0.5065386692682902
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,float16,31,0.5072213411331177
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,float16,7,0.508074680964152
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,float16,127,0.5142186482747396
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,float16,1,0.9770666758219401
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,float16,3,0.9757013320922852
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,float16,15,0.9753599961598715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,float16,31,0.9748480319976807
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,float16,63,0.5085866848627726
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,float16,63,0.974677324295044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,float16,7,0.9753599961598715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,1,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,3,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,7,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,15,0.01791999985774358
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,63,0.018432000031073887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,127,0.020992000897725422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,255,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,511,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,float16,127,0.9796266555786133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,1023,0.023210667073726654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,2047,0.03583999971548716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,4095,0.04949333270390829
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,31,0.01791999985774358
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,8191,0.07560533285140991
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,16383,0.11008000373840332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,32767,0.16674133141835532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,1,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,3,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,7,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,15,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,127,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,63,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,255,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,1023,0.027989332874615986
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,511,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,2047,0.04130133241415024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,4095,0.06980266670385997
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,31,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,8191,0.09915733337402344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,1,0.012800000607967377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,7,0.012800000607967377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,32767,0.20872533321380615
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,3,0.012458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,15,0.012800000607967377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,16383,0.14165332913398743
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,63,0.012458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,127,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,255,0.018432000031073887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,511,0.027477333943049114
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,1023,0.04130133241415024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,2047,0.06843733290831248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,31,0.012800000607967377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,4095,0.10103467106819153
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,8191,0.1360213359196981
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,16383,0.20258132616678873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,1,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,3,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,7,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,32767,0.35225598017374676
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,63,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,31,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,15,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,255,0.02611200014750163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,127,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,511,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,2047,0.09489066402117412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,1023,0.06382933259010315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,4095,0.1288533310095469
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,8191,0.19899733861287436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,32767,0.6773760318756104
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,16383,0.3573760191599528
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,1,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,7,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,15,0.012117333710193634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,3,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,63,0.012117333710193634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,127,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,31,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,255,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,511,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,1023,0.027989332874615986
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,2047,0.04164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,8191,0.09915733337402344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,16383,0.14284800489743552
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,1,0.012800000607967377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,4095,0.06911999980608623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,3,0.012629333883523941
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,7,0.012458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,15,0.012970666090647379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,32767,0.20974934101104736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,31,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,63,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,127,0.012117333710193634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,511,0.027477333943049114
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,1023,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,2047,0.06894933183987935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,4095,0.10154666503270467
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,255,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,16383,0.20292266209920248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,8191,0.13704533378283182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,1,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,3,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,7,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,32767,0.35447466373443604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,15,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,63,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,511,0.04130133241415024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,255,0.0264533335963885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,1023,0.06417066852251689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,127,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,31,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,2047,0.09574400385220845
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,4095,0.13090133666992188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,8191,0.1996799906094869
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,16383,0.3568640152613322
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,32767,0.677717367808024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,1,0.024405332903067272
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,15,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,3,0.024234667420387268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,63,0.024234667420387268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,31,0.024234667420387268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,511,0.0653653343518575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,127,0.024746666351954143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,255,0.04113066693147024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,1023,0.09130666653315227
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,7,0.024234667420387268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,4095,0.20292266209920248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,2047,0.13329066832860312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,8191,0.37836798032124835
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,16383,0.6770346959431967
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,3,0.010751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,32767,1.3129386901855469
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,63,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,31,0.012117333710193634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,7,0.010751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,255,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,511,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,1023,0.021333334346612293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,2047,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,4095,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,16383,0.0266239990790685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,8191,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,65535,0.04164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,3,0.010751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,32767,0.03345066557327906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,7,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,131071,0.059562668204307556
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,15,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,31,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,63,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,255,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,1023,0.021162666380405426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,2047,0.01791999985774358
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,4095,0.019968000551064808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,511,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,8191,0.02491733431816101
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,16383,0.026965332527955372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,32767,0.034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,65535,0.05973333120346069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,3,0.010751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,131071,0.08533333738644917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,31,0.012117333710193634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,15,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,63,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,255,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,7,0.010751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,511,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,1023,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,2047,0.01791999985774358
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,8191,0.025600001215934753
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,4095,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,16383,0.03822933385769526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,32767,0.051029334465662636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,1,0.010751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,131071,0.13107200463612875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,31,0.012117333710193634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,65535,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,15,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,63,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,255,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,1023,0.016895999511082966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,2047,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,511,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,16383,0.051541333397229515
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,8191,0.0365226666132609
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,32767,0.08345599969228108
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,4095,0.028330666323502857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,65535,0.12731732924779257
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,7,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,131071,0.21401600042978922
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,31,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,255,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,1023,0.02184533327817917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,511,0.019968000551064808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,2047,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,4095,0.020138667275508244
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,8191,0.025429333249727886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,16383,0.028330666323502857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,32767,0.03549866626660029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,63,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,65535,0.06195199986298879
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,131071,0.09045333663622539
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,31,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,255,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,511,0.020138667275508244
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,1023,0.016895999511082966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,63,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,4095,0.020992000897725422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,8191,0.02679466704527537
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,2047,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,16383,0.039594667653242745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,32767,0.051541333397229515
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,65535,0.08686932921409607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,131071,0.13363200426101685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,31,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,63,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,255,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,511,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,2047,0.020138667275508244
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,1023,0.0170666662355264
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,4095,0.02935466667016347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,8191,0.03601066768169403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,16383,0.052906667192777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,32767,0.08243200182914734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,131071,0.21248000860214233
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,7,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,15,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,31,0.012117333710193634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,63,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,65535,0.12663466731707254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,255,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,511,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,1023,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,2047,0.028501334289709728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,4095,0.03908266623814901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,8191,0.05717333157857259
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,32767,0.13431466619173685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,1,0.013141332815090815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,65535,0.21572266022364298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,131071,0.37085866928100586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,7,0.012970666090647379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,3,0.012970666090647379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,15,0.012970666090647379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,31,0.012970666090647379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,16383,0.0865280032157898
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,63,0.012970666090647379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,127,0.012970666090647379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,255,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,511,0.027647999425729115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,1023,0.04181333382924398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,2047,0.07099733253320058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,4095,0.10427733262379964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,1,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,16383,0.20770132541656494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,3,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,7,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,15,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,31,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,63,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,127,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,8191,0.13823999961217245
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,255,0.0264533335963885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,511,0.04147200038035711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,1023,0.06570666531721751
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,2047,0.09779199957847595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,4095,0.13294933239618936
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,8191,0.20172800620396933
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,3,0.025087999800841015
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,7,0.025087999800841015
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,16383,0.36061867078145343
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,15,0.025087999800841015
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,63,0.025087999800841015
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,31,0.025087999800841015
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,1,0.025087999800841015
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,127,0.025258667767047882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,255,0.04147200038035711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,511,0.06570666531721751
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,1023,0.09284266829490662
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,2047,0.13192533453305563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,4095,0.20241065820058188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,8191,0.3587413231531779
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,1,0.03737599899371465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,3,0.03737599899371465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,7,0.03737599899371465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,15,0.039936001102129616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,31,0.03737599899371465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,16383,0.6766933600107828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,63,0.03737599899371465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,127,0.03737599899371465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,255,0.05222400029500326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,511,0.07031466563542683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,1023,0.10854400197664897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,2047,0.19165867567062378
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,4095,0.34013867378234863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,8191,0.6394879817962646
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,7,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,15,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,31,0.012458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,63,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,16383,1.2431360085805256
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,511,0.03328000009059906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,1023,0.016554666062196095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,255,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,4095,0.02218666672706604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,2047,0.020309332758188248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,8191,0.028330666323502857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,32767,0.05614933371543884
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,3,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,65535,0.09011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,131071,0.13960533340771994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,16383,0.04130133241415024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,31,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,63,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,255,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,1023,0.01672533278663953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,511,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,2047,0.021162666380405426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,8191,0.03822933385769526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,16383,0.05580799778302511
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,32767,0.08703999718030293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,4095,0.031061333914597828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,65535,0.13994666934013367
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,131071,0.21708800395329794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,7,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,31,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,255,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,1023,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,511,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,63,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,4095,0.03925333420435587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,2047,0.0290133332212766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,8191,0.05905066430568695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,16383,0.08823466300964355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,32767,0.13346133629480997
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,65535,0.21572266022364298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,3,0.011776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,7,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,15,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,131071,0.37649067242940265
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,31,0.011776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,63,0.011776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,1,0.011776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,127,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,255,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,511,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,1023,0.027136000494162243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,4095,0.06382933259010315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,2047,0.040618665516376495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,8191,0.09557333588600159
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,32767,0.20599466562271118
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,1,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,65535,0.36232535044352215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,131071,0.677717367808024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,16383,0.13823999961217245
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,3,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,15,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,7,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,31,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,63,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,127,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,255,0.027477333943049114
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,1023,0.06775466601053874
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,2047,0.10018133123715718
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,4095,0.13619200388590494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,1,0.025258667767047882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,3,0.025258667767047882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,8191,0.20616533358891806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,7,0.025258667767047882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,511,0.04266666869322459
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,15,0.025258667767047882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,31,0.025258667767047882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,127,0.025429333249727886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,63,0.025429333249727886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,255,0.04249600072701772
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,511,0.06809600194295247
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,1023,0.0942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,4095,0.20480000972747803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,8191,0.36505599816640216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,1,0.038058665891488395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,7,0.03822933385769526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,3,0.03754666695992152
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,15,0.03754666695992152
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,31,0.037717332442601524
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,63,0.03822933385769526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,2047,0.13567999998728433
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,127,0.03839999934037527
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,255,0.052906667192777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,511,0.07116800049940745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,1023,0.10939733187357585
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,2047,0.1890986760457357
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,4095,0.3386026620864868
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,3,0.06843733290831248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,7,0.06843733290831248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,15,0.0682666649421056
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,8191,0.640341321627299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,31,0.0682666649421056
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,63,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,127,0.07150933146476746
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,255,0.09250133236249287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,1,0.0682666649421056
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,511,0.1250986655553182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,1023,0.19592533508936563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,2047,0.3553280035654704
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,4095,0.656554659207662
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,1,0.02628266563018163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,3,0.02628266563018163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,15,0.0264533335963885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,8191,1.265664021174113
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,31,0.0264533335963885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,63,0.02628266563018163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,127,0.02628266563018163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,511,0.06963199873765309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,255,0.04351999859015147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,1023,0.09659733374913533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,1,0.03857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,2047,0.1372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,3,0.03857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,7,0.02628266563018163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,7,0.03857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,15,0.03857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,31,0.03857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,63,0.03874133278926214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,127,0.03857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,255,0.0554666668176651
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,511,0.07338666419188182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,1,0.06929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,2047,0.1945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,7,0.06929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,3,0.06929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,15,0.06929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,31,0.06929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,1023,0.11383466919263203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,63,0.06929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,127,0.07236266632874806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,511,0.12680533528327942
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,255,0.0942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,1023,0.19985065857569376
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,2047,0.3573760191599528
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,1,0.12458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,7,0.12458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,15,0.12544000148773193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,31,0.12544000148773193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,3,0.1250986655553182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,63,0.1264639993508657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,255,0.16947199900945029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,127,0.1302186648050944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,511,0.22988800207773843
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,1023,0.3614720106124878
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,1,0.04249600072701772
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,2047,0.6625279982884725
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,15,0.04249600072701772
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,7,0.04266666869322459
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,31,0.04249600072701772
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,63,0.04232533276081085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,127,0.04249600072701772
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,255,0.059562668204307556
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,3,0.04232533276081085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,511,0.07679999868075053
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,1,0.07287466526031494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,3,0.07253333429495494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,1023,0.1204906702041626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,7,0.07236266632874806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,15,0.07355733215808868
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,31,0.07338666419188182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,127,0.07748266557852428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,255,0.10018133123715718
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,1,0.12868266304334006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,3,0.12868266304334006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,511,0.1327786644299825
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,1023,0.20428800582885742
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,7,0.1288533310095469
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,15,0.12868266304334006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,31,0.12868266304334006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,63,0.07321600119272868
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,127,0.13397333025932312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,63,0.1302186648050944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,255,0.17561600605646768
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,511,0.23586134115854898
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,1023,0.3677866856257121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,1,0.24541866779327393
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,3,0.24541866779327393
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,15,0.24576000372568765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,31,0.24678399165471396
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,63,0.2474666635195414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,127,0.25190399090449017
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,255,0.3251199920972188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,511,0.442197322845459
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,1,0.01877333347996076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,3,0.01877333347996076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,7,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,15,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,1023,0.6966613133748373
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,31,0.02184533327817917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,7,0.24541866779327393
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,63,0.025941332181294758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,127,0.025941332181294758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,255,0.02628266563018163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,1023,0.0170666662355264
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,511,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,4095,0.0315733328461647
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,2047,0.02218666672706604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,16383,0.059903999169667564
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,32767,0.08703999718030293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,1,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,65535,0.13704533378283182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,3,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,131071,0.2213546633720398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,7,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,15,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,31,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,63,0.025941332181294758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,8191,0.03857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,127,0.025941332181294758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,255,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,511,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,2047,0.030207999050617218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,1023,0.019968000551064808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,4095,0.040448000033696495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,8191,0.060415998101234436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,32767,0.13704533378283182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,1,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,65535,0.2213546633720398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,3,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,7,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,131071,0.385535995165507
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,15,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,31,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,63,0.011776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,127,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,16383,0.09028266867001851
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,255,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,511,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,1023,0.027136000494162243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,2047,0.04027733455101649
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,4095,0.06382933259010315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,8191,0.09454933802286784
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,16383,0.13909332950909933
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,1,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,3,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,65535,0.36505599816640216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,7,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,131071,0.6842026710510254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,15,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,31,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,63,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,127,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,255,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,511,0.026965332527955372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,32767,0.2032639980316162
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,2047,0.067071999112765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,1023,0.040618665516376495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,4095,0.09898666540781657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,8191,0.1353386640548706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,16383,0.20172800620396933
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,32767,0.3515733480453491
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,65535,0.64955735206604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,1,0.07543466488520305
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,3,0.07543466488520305
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,7,0.07560533285140991
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,15,0.07543466488520305
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,31,0.07594666878382365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,63,0.07645866771539052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,127,0.08396800359090169
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,255,0.10274133086204529
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,1,0.13175466656684875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,3,0.13175466656684875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,511,0.13875200351079306
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,131071,1.2530346711476643
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,7,0.13175466656684875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,15,0.13158399860064188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,31,0.13226667046546936
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,63,0.13482667009035745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,127,0.13926399747530618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,255,0.18056533734003702
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,3,0.24832000335057577
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,511,0.244053324063619
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,15,0.24968532721201578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,7,0.2491733431816101
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,31,0.2500266631444295
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,1,0.24849067131678262
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,63,0.2512213389078776
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,127,0.25565866629282635
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,255,0.3314346671104431
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,511,0.45021867752075195
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,1,0.4819626808166504
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,7,0.4838399887084961
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,3,0.4828159809112549
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,31,0.48401065667470294
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,63,0.48452266057332355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,127,0.4875946839650472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,15,0.48418132464090985
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,255,0.6328320105870565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,511,0.8644266923268636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,1,0.13858133554458618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,3,0.1384106675783793
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,7,0.1389226714769999
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,15,0.13960533340771994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,31,0.14165332913398743
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,127,0.1479680041472117
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,1,0.25565866629282635
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,7,0.2573653260866801
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,3,0.25702399015426636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,15,0.2568533420562744
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,255,0.18875734011332193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,63,0.14097066720326742
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,31,0.2571946581204732
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,63,0.25804799795150757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,127,0.26282666126887005
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,1,0.4911786715189616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,3,0.4910080035527547
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,7,0.4910080035527547
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,255,0.34201598167419434
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,31,0.49032533168792725
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,63,0.4911786715189616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,15,0.4904959996541341
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,127,0.49646933873494464
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,1,0.9601706663767496
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,3,0.9593173662821451
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,7,0.9588053226470947
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,15,0.9582933584849039
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,255,0.6463146607081095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,1,0.017237332959969837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,3,0.017749333133300144
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,31,0.9581226507822672
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,7,0.01791999985774358
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,15,0.01791999985774358
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,127,0.9623893102010092
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,31,0.019968000551064808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,63,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,127,0.020821332931518555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,255,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,511,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,63,0.9581226507822672
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,255,1.2450133164723713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,1023,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,4095,0.04130133241415024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,2047,0.030207999050617218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,8191,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,16383,0.09181867043177287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,1,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,32767,0.13823999961217245
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,3,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,7,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,15,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,31,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,63,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,127,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,511,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,1023,0.027647999425729115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,255,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,65535,0.21964800357818604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,2047,0.040789333482583366
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,4095,0.06587733328342438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,8191,0.09710933764775594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,16383,0.14148267110188803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,1,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,32767,0.20616533358891806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,3,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,65535,0.3659093379974365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,15,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,31,0.012117333710193634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,63,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,7,0.012800000607967377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,127,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,255,0.018432000031073887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,511,0.027306665976842243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,1023,0.040618665516376495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,2047,0.06809600194295247
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,8191,0.13431466619173685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,4095,0.09847467144330342
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,1,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,3,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,32767,0.34935466448465985
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,7,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,65535,0.6488746802012125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,15,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,31,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,16383,0.20053333044052124
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,127,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,63,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,511,0.040789333482583366
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,255,0.0264533335963885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,1023,0.06417066852251689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,2047,0.09454933802286784
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,4095,0.13056000073750815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,16383,0.3546453317006429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,32767,0.6756693522135416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,65535,1.3137919902801514
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,8191,0.20036266247431436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,15,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,7,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,3,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,31,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,63,0.011776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,1,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,127,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,255,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,1023,0.027306665976842243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,2047,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,511,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,4095,0.06587733328342438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,8191,0.09762133161226909
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,1,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,3,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,32767,0.2065066695213318
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,7,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,16383,0.14097066720326742
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,15,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,65535,0.3653973340988159
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,31,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,63,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,127,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,511,0.027306665976842243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,255,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,2047,0.06843733290831248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,4095,0.10035199920336406
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,8191,0.13550933202107748
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,1023,0.040618665516376495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,16383,0.20155733823776245
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,1,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,32767,0.35310932000478107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,3,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,15,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,7,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,65535,0.6500693162282308
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,63,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,127,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,31,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,255,0.02628266563018163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,511,0.04113066693147024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,1023,0.06434133152167003
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,2047,0.09591466188430786
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,4095,0.13243732849756876
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,8191,0.20036266247431436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,16383,0.35839998722076416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,7,0.010751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,15,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,65535,1.313109318415324
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,32767,0.6773760318756104
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,31,0.012117333710193634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,63,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,255,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,511,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,1023,0.020992000897725422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,2047,0.021162666380405426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,8191,0.0240639994541804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,16383,0.0266239990790685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,32767,0.03328000009059906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,4095,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,65535,0.040618665516376495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,1,0.010751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,3,0.010751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,131071,0.05870933334032694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,7,0.010751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,15,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,31,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,255,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,511,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,1023,0.020992000897725422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,63,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,2047,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,4095,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,8191,0.025087999800841015
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,16383,0.02679466704527537
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,32767,0.0341333324710528
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,65535,0.05853866537412008
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,3,0.010751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,7,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,15,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,131071,0.08362666765848796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,1,0.010751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,31,0.012117333710193634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,63,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,255,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,1023,0.016895999511082966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,511,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,2047,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,8191,0.02611200014750163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,16383,0.03942399968703588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,4095,0.020309332758188248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,32767,0.05256533126036326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,65535,0.08567466338475545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,131071,0.13329066832860312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,7,0.010751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,15,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,63,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,255,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,31,0.012117333710193634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,511,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,1023,0.021162666380405426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,2047,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,4095,0.020138667275508244
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,8191,0.025600001215934753
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,16383,0.028160000840822857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,32767,0.03532800078392029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,7,0.010751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,131071,0.0890880028406779
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,65535,0.06058666606744131
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,31,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,15,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,63,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,255,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,511,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,1023,0.016895999511082966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,4095,0.020992000897725422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,8191,0.02611200014750163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,16383,0.03908266623814901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,2047,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,32767,0.052906667192777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,65535,0.08294400076071422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,131071,0.13141333063443503
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,7,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,63,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,31,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,255,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,511,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,1023,0.016895999511082966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,2047,0.020309332758188248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,4095,0.02935466667016347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,8191,0.0365226666132609
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,16383,0.054272000988324486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,65535,0.12902399897575378
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,1,0.012458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,3,0.012117333710193634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,7,0.012458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,15,0.012800000607967377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,131071,0.2146986722946167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,32767,0.08482133348782857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,31,0.013141332815090815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,63,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,127,0.012800000607967377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,255,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,1023,0.040618665516376495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,511,0.027306665976842243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,2047,0.06911999980608623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,8191,0.1367039978504181
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,1,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,4095,0.10171733299891154
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,3,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,16383,0.20497065782546997
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,7,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,15,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,32767,0.35498666763305664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,31,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,63,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,255,0.0264533335963885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,511,0.04147200038035711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,1023,0.06502399841944377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,2047,0.09574400385220845
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,4095,0.13192533453305563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,127,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,8191,0.20121600230534872
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,1,0.024234667420387268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,16383,0.3561813433965047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,3,0.024234667420387268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,7,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,32767,0.6770346959431967
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,31,0.024405332903067272
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,63,0.024405332903067272
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,127,0.024234667420387268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,255,0.04113066693147024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,511,0.06604800124963124
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,15,0.024234667420387268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,1023,0.09335466225941975
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,2047,0.13346133629480997
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,4095,0.20155733823776245
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,8191,0.35891199111938477
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,7,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,16383,0.6775466601053873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,32767,1.3100372950236003
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,31,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,63,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,255,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,511,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,1023,0.016895999511082966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,2047,0.020309332758188248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,4095,0.02218666672706604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,8191,0.027989332874615986
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,32767,0.05495466788609823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,16383,0.040789333482583366
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,65535,0.08823466300964355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,131071,0.13755733768145242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,7,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,31,0.012458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,63,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,15,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,255,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,511,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,1023,0.016895999511082966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,2047,0.021162666380405426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,4095,0.030378667016824085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,8191,0.036864000062147774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,16383,0.05529599885145823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,32767,0.08413867155710857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,65535,0.13004799683888754
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,15,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,131071,0.21452800432840982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,31,0.012117333710193634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,255,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,511,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,63,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,1023,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,2047,0.029525332152843475
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,8191,0.05905066430568695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,4095,0.039936001102129616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,32767,0.13550933202107748
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,16383,0.08942932883898418
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,65535,0.21811199188232422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,1,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,3,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,7,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,15,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,131071,0.37836798032124835
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,31,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,63,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,255,0.027136000494162243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,127,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,511,0.04147200038035711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,1023,0.06519466638565063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,2047,0.09676800171534221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,8191,0.2034346659978231
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,4095,0.13397333025932312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,1,0.02491733431816101
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,3,0.02491733431816101
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,7,0.025087999800841015
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,16383,0.36317865053812665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,15,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,31,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,63,0.025258667767047882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,127,0.025087999800841015
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,511,0.06621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,1023,0.09318400422732036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,255,0.04147200038035711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,2047,0.1327786644299825
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,4095,0.20309333006540933
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,8191,0.3597653309504191
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,1,0.03737599899371465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,3,0.037205333511034645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,7,0.03737599899371465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,31,0.03737599899371465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,16383,0.6801066398620605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,15,0.03737599899371465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,63,0.03754666695992152
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,127,0.03754666695992152
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,255,0.05256533126036326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,511,0.07048533360163371
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,1023,0.1114453375339508
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,2047,0.1914880077044169
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,4095,0.33689598242441815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,1,0.025258667767047882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,3,0.025258667767047882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,7,0.025258667767047882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,16383,1.248085339864095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,15,0.025429333249727886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,31,0.025258667767047882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,63,0.025429333249727886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,8191,0.6398293177286783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,127,0.025258667767047882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,255,0.041984001795450844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,511,0.06690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,1023,0.09403733412424724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,2047,0.13550933202107748
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,1,0.038058665891488395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,7,0.03839999934037527
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,4095,0.20684800545374551
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,15,0.03822933385769526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,31,0.03839999934037527
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,3,0.03857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,63,0.038058665891488395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,255,0.05273599922657013
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,127,0.03839999934037527
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,511,0.07167999943097432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,1023,0.11264000336329143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,2047,0.19182932376861572
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,3,0.06843733290831248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,7,0.06843733290831248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,4095,0.33928533395131427
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,15,0.0682666649421056
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,31,0.0682666649421056
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,63,0.0682666649421056
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,1,0.06843733290831248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,127,0.0718506673971812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,255,0.09437867005666097
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,511,0.12680533528327942
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,1023,0.19848533471425375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,2047,0.35447466373443604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,1,0.03857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,4095,0.6560426553090414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,7,0.03857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,15,0.03857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,31,0.03857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,63,0.03857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,127,0.03857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,255,0.05478399991989136
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,511,0.0730453332265218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,1023,0.11212799946467082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,1,0.06929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,3,0.06929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,3,0.03857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,2047,0.19421867529551187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,7,0.06929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,15,0.06929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,31,0.06929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,63,0.06929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,127,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,255,0.09489066402117412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,1023,0.19831466674804688
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,2047,0.3566933472951253
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,1,0.12458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,3,0.12458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,7,0.12458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,15,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,31,0.12475732962290446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,63,0.1262933313846588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,127,0.13107200463612875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,511,0.2326186696688334
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,255,0.17015467087427774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,1023,0.3660800059636434
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,511,0.12800000111262003
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,2047,0.6597973505655924
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,31,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,63,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,511,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,1023,0.016895999511082966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,2047,0.022015998760859173
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,4095,0.031231999397277832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,8191,0.03754666695992152
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,16383,0.05717333157857259
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,32767,0.0865280032157898
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,65535,0.13056000073750815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,131071,0.21964800357818604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,3,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,31,0.012458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,255,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,63,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,511,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,255,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,1023,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,2047,0.02935466667016347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,4095,0.04027733455101649
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,16383,0.08874666690826416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,32767,0.13397333025932312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,1,0.011776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,65535,0.2167466680208842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,3,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,7,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,131071,0.38126933574676514
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,15,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,31,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,63,0.011776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,127,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,255,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,511,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,1023,0.027136000494162243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,2047,0.04113066693147024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,8191,0.05853866537412008
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,4095,0.06468266745408376
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,8191,0.09591466188430786
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,16383,0.13960533340771994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,32767,0.2063360015551249
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,65535,0.3657386700312297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,1,0.0730453332265218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,131071,0.681984027226766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,7,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,15,0.07253333429495494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,31,0.07321600119272868
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,63,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,127,0.07765333354473114
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,255,0.09983999530474345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,1,0.12868266304334006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,3,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,511,0.13158399860064188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,7,0.12851199507713318
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,1023,0.20258132616678873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,15,0.12868266304334006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,31,0.12868266304334006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,3,0.12851199507713318
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,63,0.1302186648050944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,255,0.17544533809026083
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,1,0.24541866779327393
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,511,0.2397866646448771
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,3,0.24524799982706705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,1023,0.3701759974161784
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,15,0.24593067169189453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,31,0.2469546596209208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,63,0.24832000335057577
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,127,0.25173334280649823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,127,0.1346560021241506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,255,0.3259733319282532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,7,0.24541866779327393
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,511,0.4466346502304077
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,1,0.13107200463612875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,1023,0.7007573445638021
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,3,0.13056000073750815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,7,0.13124266266822815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,31,0.13209600249926248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,127,0.1389226714769999
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,63,0.13448533415794373
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,1,0.24849067131678262
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,3,0.24832000335057577
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,15,0.13141333063443503
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,255,0.18107734123865762
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,7,0.24849067131678262
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,15,0.24951465924580893
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,511,0.24166399240493774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,63,0.2513920068740845
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,127,0.25463465849558514
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,31,0.24968532721201578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,1,0.4819626808166504
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,3,0.48264535268147785
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,7,0.4838399887084961
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,511,0.4575573205947876
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,15,0.48401065667470294
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,31,0.48366932074228924
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,255,0.33399466673533124
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,127,0.48878931999206543
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,63,0.4843519926071167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,1,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,3,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,7,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,255,0.6352213223775228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,31,0.021674667795499165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,255,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,127,0.025941332181294758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,63,0.025770666698614757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,511,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,511,0.8722773392995199
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,1023,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,2047,0.030720000465710957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,4095,0.040789333482583366
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,8191,0.0602453351020813
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,15,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,32767,0.1378986636797587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,16383,0.09011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,1,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,65535,0.21845332781473795
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,3,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,7,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,131071,0.37853864828745526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,31,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,127,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,63,0.011776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,255,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,511,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,1023,0.027136000494162243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,2047,0.04027733455101649
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,4095,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,8191,0.09676800171534221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,15,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,32767,0.20377600193023682
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,16383,0.13806933164596558
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,65535,0.36505599816640216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,1,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,3,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,7,0.012117333710193634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,131071,0.6818133195241293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,31,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,127,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,255,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,63,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,511,0.027136000494162243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,1023,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,2047,0.06724266707897186
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,4095,0.10035199920336406
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,8191,0.1360213359196981
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,15,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,16383,0.20172800620396933
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,32767,0.35037867228190106
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,65535,0.6483626763025919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,131071,1.2588373025258381
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,3,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,1,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,7,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,31,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,63,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,127,0.011776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,255,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,511,0.01877333347996076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,1023,0.027306665976842243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,15,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,2047,0.040618665516376495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,4095,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,8191,0.09608532985051473
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,16383,0.1384106675783793
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,32767,0.20480000972747803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,65535,0.36556800206502277
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,3,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,7,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,15,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,31,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,63,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,127,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,255,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,511,0.027306665976842243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,1023,0.040618665516376495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,131071,0.673962672551473
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,2047,0.06929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,1,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,4095,0.10188800096511841
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,8191,0.13687466581662497
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,16383,0.20206934213638306
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,32767,0.3534506559371948
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,1,0.010751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,7,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,65535,0.6504106521606445
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,15,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,31,0.012117333710193634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,63,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,255,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,511,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,131071,1.2540586789449055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,3,0.010751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,1023,0.020821332931518555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,2047,0.021162666380405426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,4095,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,8191,0.024405332903067272
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,16383,0.02611200014750163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,32767,0.03242666771014532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,1,0.010751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,131071,0.056832000613212585
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,7,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,15,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,31,0.012117333710193634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,63,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,65535,0.040106666584809623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,255,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,511,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,1023,0.02065066620707512
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,2047,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,4095,0.019968000551064808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,8191,0.024746666351954143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,16383,0.027306665976842243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,1,0.010581333190202713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,65535,0.059562668204307556
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,131071,0.08772266904513042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,15,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,31,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,32767,0.034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,63,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,255,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,511,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,1023,0.020992000897725422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,2047,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,4095,0.019968000551064808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,16383,0.027136000494162243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,32767,0.03515733281771342
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,65535,0.059903999169667564
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,1,0.010581333190202713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,3,0.010751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,131071,0.08567466338475545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,7,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,15,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,8191,0.025258667767047882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,31,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,63,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,255,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,511,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,1023,0.016895999511082966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,2047,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,8191,0.027136000494162243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,16383,0.040106666584809623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,32767,0.051882664362589516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,65535,0.08618666728337605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,1,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,3,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,131071,0.13499733805656433
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,4095,0.020992000897725422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,7,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,15,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,31,0.012117333710193634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,63,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,127,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,255,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,511,0.027306665976842243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,4095,0.10086400310198466
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,2047,0.0682666649421056
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,8191,0.13704533378283182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,16383,0.20241065820058188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,1,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,32767,0.35259731610616046
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,3,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,7,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,1023,0.040448000033696495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,15,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,31,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,63,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,65535,0.655189315478007
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,127,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,255,0.02628266563018163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,511,0.040789333482583366
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,2047,0.0962559978167216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,4095,0.1327786644299825
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,8191,0.2027519941329956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,16383,0.3587413231531779
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,32767,0.6768639882405599
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,7,0.010751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,1023,0.06502399841944377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,31,0.012117333710193634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,63,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,65535,1.3100372950236003
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,255,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,511,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,2047,0.020309332758188248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,4095,0.02218666672706604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,8191,0.027477333943049114
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,16383,0.04027733455101649
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,32767,0.05256533126036326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,65535,0.08550399541854858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,131071,0.13414399822553
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,7,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,15,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,63,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,31,0.012458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,1023,0.01672533278663953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,511,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,255,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,1023,0.016895999511082966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,2047,0.022015998760859173
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,4095,0.03089066594839096
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,8191,0.038058665891488395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,32767,0.08721066514650981
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,1,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,65535,0.13056000073750815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,3,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,131071,0.21555199225743613
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,7,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,15,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,31,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,63,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,127,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,255,0.0264533335963885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,16383,0.05717333157857259
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,511,0.04113066693147024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,1023,0.06434133152167003
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,2047,0.09659733374913533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,4095,0.13175466656684875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,8191,0.20189867417017618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,3,0.024405332903067272
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,16383,0.35839998722076416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,7,0.024234667420387268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,15,0.024234667420387268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,31,0.024746666351954143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,32767,0.6784000396728516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,63,0.024234667420387268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,127,0.024234667420387268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,255,0.04113066693147024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,511,0.06604800124963124
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,1023,0.0936959981918335
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,1,0.024405332903067272
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,2047,0.13482667009035745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,4095,0.20292266209920248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,1,0.025087999800841015
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,8191,0.36078933874766034
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,3,0.024746666351954143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,16383,0.6768639882405599
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,15,0.025087999800841015
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,31,0.02491733431816101
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,63,0.025087999800841015
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,32767,1.312597354253133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,127,0.02491733431816101
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,255,0.04147200038035711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,511,0.06587733328342438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,1023,0.09250133236249287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,2047,0.13431466619173685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,7,0.025087999800841015
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,1,0.03703466554482778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,3,0.03737599899371465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,4095,0.2034346659978231
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,7,0.03737599899371465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,8191,0.36113067468007404
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,15,0.03737599899371465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,31,0.03737599899371465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,127,0.03737599899371465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,255,0.05341866612434387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,511,0.07150933146476746
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,1023,0.1109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,2047,0.1904639999071757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,4095,0.3399680058161418
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,1,0.03788800040880839
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,3,0.038058665891488395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,7,0.03839999934037527
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,63,0.03737599899371465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,15,0.038058665891488395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,8191,0.6377813418706259
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,31,0.03857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,63,0.03857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,127,0.03839999934037527
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,255,0.05341866612434387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,511,0.0721919983625412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,1,0.0682666649421056
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,2047,0.19182932376861572
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,3,0.0682666649421056
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,7,0.0682666649421056
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,15,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,31,0.0682666649421056
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,4095,0.3444053332010905
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,63,0.06894933183987935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,127,0.07099733253320058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,255,0.0936959981918335
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,511,0.1276586651802063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,1023,0.10769066214561462
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,1023,0.20087466637293497
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,1,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,2047,0.3573760191599528
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,7,0.010751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,15,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,4095,0.65228799978892
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,255,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,1023,0.016895999511082966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,511,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,2047,0.021162666380405426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,4095,0.03054933249950409
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,8191,0.03754666695992152
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,16383,0.057002668579419456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,32767,0.08499200145403545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,31,0.012117333710193634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,65535,0.1288533310095469
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,131071,0.2152106761932373
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,31,0.012117333710193634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,255,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,511,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,1023,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,2047,0.029696000119050343
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,4095,0.040618665516376495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,8191,0.05922133227189382
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,16383,0.08925867080688477
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,32767,0.13687466581662497
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,65535,0.21811199188232422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,63,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,1,0.06929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,3,0.06929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,7,0.06929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,131071,0.38331735134124756
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,15,0.06946133573849995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,31,0.06929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,63,0.06929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,255,0.09471999605496724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,511,0.1281706690788269
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,1,0.12458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,1023,0.19814399878184
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,3,0.12458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,7,0.12475732962290446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,15,0.12544000148773193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,2047,0.3548159996668498
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,31,0.12475732962290446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,127,0.0718506673971812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,63,0.12612266341845194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,127,0.13141333063443503
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,255,0.1718613306681315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,511,0.23586134115854898
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,1,0.12868266304334006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,1023,0.36642134189605713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,7,0.12851199507713318
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,15,0.12851199507713318
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,31,0.12851199507713318
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,2047,0.6574079990386963
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,63,0.13056000073750815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,127,0.13431466619173685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,3,0.12851199507713318
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,255,0.176639993985494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,1,0.24524799982706705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,3,0.24541866779327393
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,7,0.24524799982706705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,511,0.2373973329861959
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,15,0.24541866779327393
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,63,0.24780799945195517
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,1023,0.3710293372472127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,127,0.2510506709416707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,255,0.328874667485555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,15,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,31,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,31,0.24661332368850708
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,511,0.45448533693949383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,255,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,1023,0.019968000551064808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,511,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,1023,0.7109973430633545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,4095,0.03976533313592275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,8191,0.05905066430568695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,16383,0.0885759989420573
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,32767,0.13482667009035745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,1,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,65535,0.2167466680208842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,3,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,7,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,2047,0.02935466667016347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,15,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,131071,0.3742719888687134
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,31,0.011776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,63,0.011776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,127,0.011946666985750198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,255,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,511,0.01877333347996076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,2047,0.040448000033696495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,4095,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,8191,0.09727999567985535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,16383,0.13960533340771994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,32767,0.20514132579167685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,65535,0.3653973340988159
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,131071,0.6795946756998698
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,1023,0.027136000494162243
