framework,version,device,op_name,kernel_source,batch_size,isl,num_heads,num_key_value_heads,head_dim,window_size,beam_width,attn_dtype,kv_cache_dtype,step,latency
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,96,2,128,0,1,float16,float16,0,176.02714029947916
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,96,1,128,0,1,float16,float16,0,176.96563720703125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,96,4,128,0,1,float16,float16,0,178.0526123046875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,96,1,128,0,1,float16,float16,0,89.5810546875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,96,8,128,0,1,float16,float16,0,176.4459431966146
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,96,2,128,0,1,float16,float16,0,89.94798787434895
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,96,96,128,0,1,float16,float16,0,89.8693135579427
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,96,96,128,0,1,float16,float16,0,176.3048095703125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,96,1,128,0,1,float16,float16,0,43.9552001953125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,96,4,128,0,1,float16,float16,0,88.8065694173177
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,96,8,128,0,1,float16,float16,0,89.72202555338542
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,96,96,128,0,1,float16,float16,0,45.11658732096354
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,96,2,128,0,1,float16,float16,0,44.373504638671875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,96,4,128,0,1,float16,float16,0,44.66773478190104
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,96,8,128,0,1,float16,float16,0,44.98739115397135
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,96,1,128,0,1,float16,float16,0,100.48853556315105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,96,2,128,0,1,float16,float16,0,101.54359944661458
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,96,4,128,0,1,float16,float16,0,101.63302612304688
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,12288,96,1,128,0,1,float16,float16,0,201.7264607747396
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,96,1,128,0,1,float16,float16,0,50.80644226074219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,12288,96,2,128,0,1,float16,float16,0,199.1052042643229
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,96,8,128,0,1,float16,float16,0,100.53956095377605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,96,2,128,0,1,float16,float16,0,50.64362589518229
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,12288,96,4,128,0,1,float16,float16,0,202.20245361328125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,96,96,128,0,1,float16,float16,0,51.457194010416664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,12288,96,8,128,0,1,float16,float16,0,199.48492431640625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,96,1,128,0,1,float16,float16,0,25.706153869628906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,96,4,128,0,1,float16,float16,0,50.88170878092448
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,96,8,128,0,1,float16,float16,0,50.0848642985026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,96,2,128,0,1,float16,float16,0,25.798655192057293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,96,96,128,0,1,float16,float16,0,102.92002360026042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,96,4,128,0,1,float16,float16,0,25.916414896647137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,96,8,128,0,1,float16,float16,0,25.53412373860677
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,96,96,128,0,1,float16,float16,0,25.71502939860026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,96,1,128,0,1,float16,float16,0,71.58408610026042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,96,2,128,0,1,float16,float16,0,70.4179178873698
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,96,4,128,0,1,float16,float16,0,70.91729227701823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,10240,96,1,128,0,1,float16,float16,0,141.68490600585938
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,96,1,128,0,1,float16,float16,0,35.2170664469401
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,10240,96,2,128,0,1,float16,float16,0,142.22250366210938
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,10240,96,4,128,0,1,float16,float16,0,139.82190958658853
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,96,8,128,0,1,float16,float16,0,71.72352091471355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,96,2,128,0,1,float16,float16,0,35.5956064860026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,10240,96,8,128,0,1,float16,float16,0,141.8740030924479
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,96,4,128,0,1,float16,float16,0,35.26263427734375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,96,96,128,0,1,float16,float16,0,35.611307779947914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,96,1,128,0,1,float16,float16,0,18.139989217122395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,96,8,128,0,1,float16,float16,0,35.87566884358724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,96,2,128,0,1,float16,float16,0,18.07786687215169
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,96,4,128,0,1,float16,float16,0,18.197675069173176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,96,96,128,0,1,float16,float16,0,18.047658284505207
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,96,8,128,0,1,float16,float16,0,18.034517923990887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,96,96,128,0,1,float16,float16,0,72.95624287923177
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,96,1,128,0,1,float16,float16,0,91.01619466145833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,96,2,128,0,1,float16,float16,0,91.63075764973958
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,96,4,128,0,1,float16,float16,0,92.69435628255208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,96,1,128,0,1,float16,float16,0,46.15270487467448
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,96,8,128,0,1,float16,float16,0,91.2013651529948
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,96,2,128,0,1,float16,float16,0,46.32268778483073
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,96,96,128,0,1,float16,float16,0,46.687744140625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,96,4,128,0,1,float16,float16,0,45.10156758626302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,96,96,128,0,1,float16,float16,0,94.00712076822917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,96,8,128,0,1,float16,float16,0,46.261077880859375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,96,1,128,0,1,float16,float16,0,22.770347595214844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,96,96,128,0,1,float16,float16,0,23.25401560465495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,96,2,128,0,1,float16,float16,0,23.172266642252605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,96,4,128,0,1,float16,float16,0,23.239168802897137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,96,1,128,0,1,float16,float16,0,11.929941813151041
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,96,8,128,0,1,float16,float16,0,22.865577697753906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,96,2,128,0,1,float16,float16,0,11.957930246988932
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,96,96,128,0,1,float16,float16,0,11.862186431884766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,96,4,128,0,1,float16,float16,0,11.941205342610678
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,96,8,128,0,1,float16,float16,0,11.642197926839193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,96,1,128,0,1,float16,float16,0,52.17467753092448
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,96,4,128,0,1,float16,float16,0,53.21454874674479
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,96,2,128,0,1,float16,float16,0,52.83447265625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,96,1,128,0,1,float16,float16,0,26.42602793375651
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,96,8,128,0,1,float16,float16,0,52.16119384765625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,96,2,128,0,1,float16,float16,0,26.437291463216145
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,96,96,128,0,1,float16,float16,0,27.3080317179362
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,96,96,128,0,1,float16,float16,0,54.6001942952474
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,96,4,128,0,1,float16,float16,0,25.86572774251302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,96,1,128,0,1,float16,float16,0,13.444608052571615
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,96,2,128,0,1,float16,float16,0,13.147135416666666
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,96,8,128,0,1,float16,float16,0,26.46613311767578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,96,4,128,0,1,float16,float16,0,13.457407633463541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,96,96,128,0,1,float16,float16,0,13.66647466023763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,96,8,128,0,1,float16,float16,0,13.124095916748047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,96,1,128,0,1,float16,float16,0,6.936234792073567
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,96,96,128,0,1,float16,float16,0,7.045461018880208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,96,2,128,0,1,float16,float16,0,6.926677068074544
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,96,4,128,0,1,float16,float16,0,6.740479787190755
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,96,8,128,0,1,float16,float16,0,6.908416112263997
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,96,1,128,0,1,float16,float16,0,49.459370930989586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,96,2,128,0,1,float16,float16,0,48.819539388020836
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,96,4,128,0,1,float16,float16,0,49.234771728515625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,96,1,128,0,1,float16,float16,0,24.288597106933594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,96,8,128,0,1,float16,float16,0,49.71707661946615
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,96,2,128,0,1,float16,float16,0,24.518826802571613
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,96,96,128,0,1,float16,float16,0,25.275733947753906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,96,96,128,0,1,float16,float16,0,50.764628092447914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,96,4,128,0,1,float16,float16,0,24.513193766276043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,96,1,128,0,1,float16,float16,0,12.19549814860026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,96,8,128,0,1,float16,float16,0,24.518145243326824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,96,2,128,0,1,float16,float16,0,12.328788757324219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,96,4,128,0,1,float16,float16,0,12.144981384277344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,96,96,128,0,1,float16,float16,0,12.616704305013021
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,96,8,128,0,1,float16,float16,0,12.2881711324056
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,96,1,128,0,1,float16,float16,0,6.253909428914388
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,96,2,128,0,1,float16,float16,0,6.131370544433594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,96,4,128,0,1,float16,float16,0,6.1281280517578125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,96,8,128,0,1,float16,float16,0,6.265685399373372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,96,1,128,0,1,float16,float16,0,3.197781244913737
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,96,96,128,0,1,float16,float16,0,6.4651947021484375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,96,2,128,0,1,float16,float16,0,3.2179199854532876
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,96,96,128,0,1,float16,float16,0,3.280384063720703
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,96,8,128,0,1,float16,float16,0,3.2602453231811523
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,96,4,128,0,1,float16,float16,0,3.2075093587239585
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,96,1,128,0,1,float16,float16,0,28.975616455078125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,96,4,128,0,1,float16,float16,0,28.384938557942707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,96,2,128,0,1,float16,float16,0,28.93516794840495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,96,1,128,0,1,float16,float16,0,14.215680440266928
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,96,8,128,0,1,float16,float16,0,28.993024190266926
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,96,2,128,0,1,float16,float16,0,14.213461558024088
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,96,96,128,0,1,float16,float16,0,14.895103454589844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,96,4,128,0,1,float16,float16,0,14.391979217529297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,96,96,128,0,1,float16,float16,0,30.595072428385418
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,96,8,128,0,1,float16,float16,0,14.23854955037435
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,96,1,128,0,1,float16,float16,0,7.1990610758463545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,96,2,128,0,1,float16,float16,0,7.273642857869466
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,96,4,128,0,1,float16,float16,0,7.189333597819011
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,96,96,128,0,1,float16,float16,0,7.410858790079753
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,96,8,128,0,1,float16,float16,0,7.263231913248698
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,96,1,128,0,1,float16,float16,0,3.6299091974894204
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,96,2,128,0,1,float16,float16,0,3.603626569112142
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,96,96,128,0,1,float16,float16,0,3.7949441274007163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,96,4,128,0,1,float16,float16,0,3.6406612396240234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,96,8,128,0,1,float16,float16,0,3.658752123514811
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,96,2,128,0,1,float16,float16,0,1.9360426266988118
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,96,1,128,0,1,float16,float16,0,1.9420159657796223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,96,4,128,0,1,float16,float16,0,1.9488426844278972
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,96,96,128,0,1,float16,float16,0,1.960960070292155
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,96,8,128,0,1,float16,float16,0,1.9379199345906575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,96,2,128,0,1,float16,float16,0,27.781461079915363
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,96,1,128,0,1,float16,float16,0,27.790506998697918
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,96,4,128,0,1,float16,float16,0,28.029953002929688
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,96,1,128,0,1,float16,float16,0,13.621077219645182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,96,8,128,0,1,float16,float16,0,27.71148681640625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,96,2,128,0,1,float16,float16,0,13.679957071940104
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,96,96,128,0,1,float16,float16,0,14.840831756591797
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,96,4,128,0,1,float16,float16,0,13.446144104003906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,96,96,128,0,1,float16,float16,0,29.68866221110026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,96,8,128,0,1,float16,float16,0,13.811200459798178
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,96,1,128,0,1,float16,float16,0,6.743381500244141
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,96,2,128,0,1,float16,float16,0,6.757546742757161
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,96,96,128,0,1,float16,float16,0,7.4202454884847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,96,4,128,0,1,float16,float16,0,6.831786473592122
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,96,8,128,0,1,float16,float16,0,6.753109614054362
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,96,2,128,0,1,float16,float16,0,3.4210131963094077
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,96,1,128,0,1,float16,float16,0,3.405994733174642
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,96,96,128,0,1,float16,float16,0,3.64356263478597
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,96,2,128,0,1,float16,float16,0,1.7832959493001301
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,96,4,128,0,1,float16,float16,0,3.434666633605957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,96,4,128,0,1,float16,float16,0,1.7832959493001301
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,96,1,128,0,1,float16,float16,0,1.7868800163269043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,96,1,128,0,1,float16,float16,0,0.9821866353352865
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,96,96,128,0,1,float16,float16,0,1.8534399668375652
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,96,4,128,0,1,float16,float16,0,0.986794630686442
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,96,8,128,0,1,float16,float16,0,3.3995094299316406
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,96,8,128,0,1,float16,float16,0,1.7827839851379395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,96,2,128,0,1,float16,float16,0,0.9825279712677002
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,96,96,128,0,1,float16,float16,0,0.9931093056996664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,96,8,128,0,1,float16,float16,0,0.9844053586324056
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,96,2,128,0,1,float16,float16,0,16.864938100179035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,96,1,128,0,1,float16,float16,0,16.735572814941406
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,96,4,128,0,1,float16,float16,0,16.944981892903645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,96,1,128,0,1,float16,float16,0,8.233472188313803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,96,8,128,0,1,float16,float16,0,16.873130798339844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,96,2,128,0,1,float16,float16,0,8.168959935506185
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,96,96,128,0,1,float16,float16,0,9.133567810058594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,96,4,128,0,1,float16,float16,0,8.195413589477539
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,96,96,128,0,1,float16,float16,0,18.31167984008789
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,96,8,128,0,1,float16,float16,0,8.30463981628418
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,96,1,128,0,1,float16,float16,0,4.085247993469238
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,96,2,128,0,1,float16,float16,0,4.060160001118978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,96,4,128,0,1,float16,float16,0,4.057258605957031
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,96,96,128,0,1,float16,float16,0,4.52625052134196
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,96,8,128,0,1,float16,float16,0,4.0820051829020185
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,96,1,128,0,1,float16,float16,0,2.077866713205973
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,96,2,128,0,1,float16,float16,0,2.0775252978006997
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,96,4,128,0,1,float16,float16,0,2.077866713205973
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,96,96,128,0,1,float16,float16,0,2.2341972986857095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,96,8,128,0,1,float16,float16,0,2.0833279291788735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,96,1,128,0,1,float16,float16,0,1.1009706656138103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,96,96,128,0,1,float16,float16,0,1.1557546456654866
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,96,4,128,0,1,float16,float16,0,1.1037013530731201
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,96,2,128,0,1,float16,float16,0,1.1013120015462239
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,96,8,128,0,1,float16,float16,0,1.1048959891001384
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,96,1,128,0,1,float16,float16,0,0.6224213441212972
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,96,96,128,0,1,float16,float16,0,0.6341973145802816
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,96,4,128,0,1,float16,float16,0,0.6202026605606079
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,96,2,128,0,1,float16,float16,0,0.6219093402226766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,96,8,128,0,1,float16,float16,0,0.6227626800537109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,96,1,128,0,1,float16,float16,0,16.23688507080078
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,96,2,128,0,1,float16,float16,0,16.017920176188152
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,96,4,128,0,1,float16,float16,0,16.338943481445312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,96,1,128,0,1,float16,float16,0,7.988565444946289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,96,8,128,0,1,float16,float16,0,16.51421864827474
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,96,2,128,0,1,float16,float16,0,8.038229624430338
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,96,96,128,0,1,float16,float16,0,9.41653315226237
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,96,4,128,0,1,float16,float16,0,8.207701365152994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,96,96,128,0,1,float16,float16,0,18.831872304280598
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,96,1,128,0,1,float16,float16,0,3.9835306803385415
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,96,8,128,0,1,float16,float16,0,8.12390391031901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,96,2,128,0,1,float16,float16,0,3.958613395690918
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,96,4,128,0,1,float16,float16,0,4.029269218444824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,96,96,128,0,1,float16,float16,0,4.749141375223796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,96,1,128,0,1,float16,float16,0,2.0027732849121094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,96,8,128,0,1,float16,float16,0,4.018517176310222
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,96,2,128,0,1,float16,float16,0,1.9995306332906086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,96,96,128,0,1,float16,float16,0,2.3661227226257324
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,96,4,128,0,1,float16,float16,0,2.0147199630737305
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,96,1,128,0,1,float16,float16,0,1.0151253541310628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,96,8,128,0,1,float16,float16,0,2.015061378479004
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,96,2,128,0,1,float16,float16,0,1.0199039777119954
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,96,4,128,0,1,float16,float16,0,1.0246826807657878
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,96,1,128,0,1,float16,float16,0,0.5309439897537231
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,96,8,128,0,1,float16,float16,0,1.0320213635762532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,96,2,128,0,1,float16,float16,0,0.529578685760498
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,96,96,128,0,1,float16,float16,0,0.6089386542638143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,96,4,128,0,1,float16,float16,0,0.5335040092468262
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,96,96,128,0,1,float16,float16,0,1.1803306738535564
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,96,8,128,0,1,float16,float16,0,0.5350399812062582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,96,1,128,0,1,float16,float16,0,0.283135990301768
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,96,2,128,0,1,float16,float16,0,0.28194133440653485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,96,96,128,0,1,float16,float16,0,0.32477867603302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,96,4,128,0,1,float16,float16,0,0.2851840058962504
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,96,8,128,0,1,float16,float16,0,0.2874026695887248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,96,1,128,0,1,float16,float16,0,10.919935862223307
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,96,2,128,0,1,float16,float16,0,10.994859059651693
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,96,4,128,0,1,float16,float16,0,11.123199462890625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,96,1,128,0,1,float16,float16,0,5.395456314086914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,96,2,128,0,1,float16,float16,0,5.388458887736003
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,96,8,128,0,1,float16,float16,0,11.246592203776041
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,96,96,128,0,1,float16,float16,0,6.92036247253418
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,96,4,128,0,1,float16,float16,0,5.505706787109375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,96,8,128,0,1,float16,float16,0,5.553152084350586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,96,96,128,0,1,float16,float16,0,13.802837371826172
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,96,1,128,0,1,float16,float16,0,2.685098648071289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,96,2,128,0,1,float16,float16,0,2.6791254679361978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,96,4,128,0,1,float16,float16,0,2.7275946935017905
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,96,96,128,0,1,float16,float16,0,3.4715305964152017
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,96,8,128,0,1,float16,float16,0,2.7347625096639
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,96,1,128,0,1,float16,float16,0,1.3470719655354817
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,96,2,128,0,1,float16,float16,0,1.3597013155619304
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,96,4,128,0,1,float16,float16,0,1.3653333981831868
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,96,96,128,0,1,float16,float16,0,1.7220266660054524
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,96,8,128,0,1,float16,float16,0,1.3707946141560872
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,96,1,128,0,1,float16,float16,0,0.6905173460642496
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,96,2,128,0,1,float16,float16,0,0.6901760101318359
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,96,96,128,0,1,float16,float16,0,0.854698657989502
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,96,4,128,0,1,float16,float16,0,0.6959786415100098
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,96,1,128,0,1,float16,float16,0,0.3595946629842122
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,96,8,128,0,1,float16,float16,0,0.7007573445638021
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,96,2,128,0,1,float16,float16,0,0.3592533270517985
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,96,96,128,0,1,float16,float16,0,0.4403200149536133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,96,4,128,0,1,float16,float16,0,0.3633493185043335
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,96,1,128,0,1,float16,float16,0,0.19285333156585693
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,96,8,128,0,1,float16,float16,0,0.3660800059636434
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,96,96,128,0,1,float16,float16,0,0.23398399353027344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,96,2,128,0,1,float16,float16,0,0.1930239995320638
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,96,4,128,0,1,float16,float16,0,0.19438934326171875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,96,8,128,0,1,float16,float16,0,0.19643733898798624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,96,2,128,0,1,float16,float16,0,0.106495996316274
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,96,96,128,0,1,float16,float16,0,0.1293653349081675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,96,1,128,0,1,float16,float16,0,0.105813334385554
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,96,4,128,0,1,float16,float16,0,0.10752000411351521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,96,8,128,0,1,float16,float16,0,0.10820266604423523
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,96,1,128,0,1,float16,float16,0,4.048384030659993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,96,2,128,0,1,float16,float16,0,4.065279960632324
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,256,96,1,128,0,1,float16,float16,0,8.228864034016928
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,256,96,2,128,0,1,float16,float16,0,8.230400085449219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,256,96,8,128,0,1,float16,float16,0,8.57907231648763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,96,1,128,0,1,float16,float16,0,2.0360533396402993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,96,4,128,0,1,float16,float16,0,4.195839881896973
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,96,2,128,0,1,float16,float16,0,2.0386133193969727
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,256,96,4,128,0,1,float16,float16,0,8.388437271118164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,96,8,128,0,1,float16,float16,0,4.200448036193848
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,96,1,128,0,1,float16,float16,0,1.0246826807657878
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,96,96,128,0,1,float16,float16,0,5.648725509643555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,96,4,128,0,1,float16,float16,0,2.0850346883138022
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,96,8,128,0,1,float16,float16,0,2.0980052947998047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,96,2,128,0,1,float16,float16,0,1.030997355779012
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,96,4,128,0,1,float16,float16,0,1.0449919700622559
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,96,96,128,0,1,float16,float16,0,2.8248745600382485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,96,1,128,0,1,float16,float16,0,0.5271893342336019
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,96,8,128,0,1,float16,float16,0,1.0484053293863933
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,96,96,128,0,1,float16,float16,0,1.4354772567749023
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,96,2,128,0,1,float16,float16,0,0.5282133420308431
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,96,4,128,0,1,float16,float16,0,0.53111465771993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,96,8,128,0,1,float16,float16,0,0.53657599290212
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,96,96,128,0,1,float16,float16,0,0.7359146277109782
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,96,1,128,0,1,float16,float16,0,0.2739199995994568
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,96,2,128,0,1,float16,float16,0,0.27460267146428424
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,96,4,128,0,1,float16,float16,0,0.2792106668154399
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,96,8,128,0,1,float16,float16,0,0.28091732660929364
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,96,96,128,0,1,float16,float16,0,0.38126933574676514
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,96,1,128,0,1,float16,float16,0,0.1469439963499705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,96,2,128,0,1,float16,float16,0,0.14762666821479797
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,96,96,128,0,1,float16,float16,0,0.19797333081563315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,96,4,128,0,1,float16,float16,0,0.14830933014551798
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,96,1,128,0,1,float16,float16,0,0.08157866696516673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,96,8,128,0,1,float16,float16,0,0.15121066570281982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,96,2,128,0,1,float16,float16,0,0.08106666803359985
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,96,96,128,0,1,float16,float16,0,0.10803199807802837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,96,4,128,0,1,float16,float16,0,0.0820906658967336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,96,8,128,0,1,float16,float16,0,0.08243200182914734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,96,96,128,0,1,float16,float16,0,0.05734399954477946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,96,1,128,0,1,float16,float16,0,0.04983466863632202
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,96,2,128,0,1,float16,float16,0,0.04966400067011515
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,96,4,128,0,1,float16,float16,0,0.04898133377234141
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,96,8,128,0,1,float16,float16,0,0.04949333270390829
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,96,1,128,0,1,float16,float16,0,1.7486507097880046
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,96,2,128,0,1,float16,float16,0,1.7481387456258137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,96,4,128,0,1,float16,float16,0,1.8066773414611816
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,96,1,128,0,1,float16,float16,0,0.8830293019612631
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,128,96,2,128,0,1,float16,float16,0,3.530581474304199
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,128,96,1,128,0,1,float16,float16,0,3.527850786844889
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,128,96,4,128,0,1,float16,float16,0,3.6355412801106772
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,128,96,8,128,0,1,float16,float16,0,3.6538025538126626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,96,8,128,0,1,float16,float16,0,1.8206720352172852
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,96,2,128,0,1,float16,float16,0,0.882858673731486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,96,96,128,0,1,float16,float16,0,1.3424639701843262
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,96,4,128,0,1,float16,float16,0,0.9009493192036947
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,96,1,128,0,1,float16,float16,0,0.4510720173517863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,96,2,128,0,1,float16,float16,0,0.4532906611760457
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,96,8,128,0,1,float16,float16,0,0.912384033203125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,96,4,128,0,1,float16,float16,0,0.4580693244934082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,96,1,128,0,1,float16,float16,0,0.23432532946268717
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,96,2,128,0,1,float16,float16,0,0.23432532946268717
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,96,4,128,0,1,float16,float16,0,0.23825067281723022
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,96,8,128,0,1,float16,float16,0,0.4643839995066325
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,96,96,128,0,1,float16,float16,0,0.7053653399149576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,96,96,128,0,1,float16,float16,0,2.6668373743693032
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,96,96,128,0,1,float16,float16,0,0.35310932000478107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,96,8,128,0,1,float16,float16,0,0.23995733261108398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,96,1,128,0,1,float16,float16,0,0.12339199582735698
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,96,2,128,0,1,float16,float16,0,0.12475732962290446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,96,8,128,0,1,float16,float16,0,0.12782933314641318
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,96,4,128,0,1,float16,float16,0,0.12612266341845194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,96,96,128,0,1,float16,float16,0,0.18756266434987387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,96,1,128,0,1,float16,float16,0,0.06690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,96,2,128,0,1,float16,float16,0,0.06638933221499126
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,96,4,128,0,1,float16,float16,0,0.06673066814740498
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,96,8,128,0,1,float16,float16,0,0.0679253339767456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,96,1,128,0,1,float16,float16,0,0.04027733455101649
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,96,96,128,0,1,float16,float16,0,0.10410666465759277
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,96,96,128,0,1,float16,float16,0,0.04642133414745331
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,96,2,128,0,1,float16,float16,0,0.03976533313592275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,96,4,128,0,1,float16,float16,0,0.038912000755469
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,96,8,128,0,1,float16,float16,0,0.038912000755469
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,96,96,128,0,1,float16,float16,0,0.026965332527955372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,96,1,128,0,1,float16,float16,0,0.025941332181294758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,96,2,128,0,1,float16,float16,0,0.025429333249727886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,96,8,128,0,1,float16,float16,0,0.025941332181294758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,96,4,128,0,1,float16,float16,0,0.025258667767047882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,96,1,128,0,1,float16,float16,0,0.8432640234629313
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,96,2,128,0,1,float16,float16,0,0.8772266705830892
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,96,4,128,0,1,float16,float16,0,0.9222826957702637
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,64,96,1,128,0,1,float16,float16,0,1.6670719782511394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,64,96,2,128,0,1,float16,float16,0,1.7488212585449219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,64,96,4,128,0,1,float16,float16,0,1.8607786496480305
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,64,96,8,128,0,1,float16,float16,0,1.8834773699442546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,96,8,128,0,1,float16,float16,0,0.942250649134318
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,96,2,128,0,1,float16,float16,0,0.4551680088043213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,96,1,128,0,1,float16,float16,0,0.44253865877787274
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,96,4,128,0,1,float16,float16,0,0.47837865352630615
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,96,96,128,0,1,float16,float16,0,1.3569706281026204
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,96,96,128,0,1,float16,float16,0,0.688981294631958
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,96,8,128,0,1,float16,float16,0,0.48452266057332355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,96,1,128,0,1,float16,float16,0,0.23347200949986777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,96,2,128,0,1,float16,float16,0,0.2387626568476359
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,96,96,128,0,1,float16,float16,0,0.36351998647054035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,96,1,128,0,1,float16,float16,0,0.15035733580589294
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,96,8,128,0,1,float16,float16,0,0.2505386670430501
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,96,2,128,0,1,float16,float16,0,0.1230506698290507
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,96,4,128,0,1,float16,float16,0,0.12663466731707254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,96,96,128,0,1,float16,float16,0,0.1955839991569519
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,96,4,128,0,1,float16,float16,0,0.24439465999603271
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,96,1,128,0,1,float16,float16,0,0.06502399841944377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,96,8,128,0,1,float16,float16,0,0.13056000073750815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,96,96,128,0,1,float16,float16,0,0.10001066327095032
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,96,2,128,0,1,float16,float16,0,0.06502399841944377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,96,4,128,0,1,float16,float16,0,0.06587733328342438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,96,8,128,0,1,float16,float16,0,0.06690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,96,96,128,0,1,float16,float16,0,0.045909335215886436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,96,2,128,0,1,float16,float16,0,0.03874133278926214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,96,4,128,0,1,float16,float16,0,0.03839999934037527
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,96,8,128,0,1,float16,float16,0,0.038058665891488395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,96,1,128,0,1,float16,float16,0,0.037205333511034645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,96,1,128,0,1,float16,float16,0,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,96,2,128,0,1,float16,float16,0,0.02491733431816101
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,96,96,128,0,1,float16,float16,0,0.025258667767047882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,96,4,128,0,1,float16,float16,0,0.025087999800841015
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,96,8,128,0,1,float16,float16,0,0.024746666351954143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,96,96,128,0,1,float16,float16,0,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,96,1,128,0,1,float16,float16,0,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,96,8,128,0,1,float16,float16,0,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,96,4,128,0,1,float16,float16,0,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,96,2,128,0,1,float16,float16,0,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,96,1,128,0,1,float16,float16,0,0.49561599890391034
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,96,2,128,0,1,float16,float16,0,0.4968106746673584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,32,96,1,128,0,1,float16,float16,0,0.9675093491872152
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,32,96,2,128,0,1,float16,float16,0,0.9779199759165446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,32,96,4,128,0,1,float16,float16,0,0.9896960258483887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,32,96,8,128,0,1,float16,float16,0,1.0006186962127686
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,96,4,128,0,1,float16,float16,0,0.5034666856129965
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,96,1,128,0,1,float16,float16,0,0.2553173303604126
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,96,8,128,0,1,float16,float16,0,0.5082453489303589
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,96,2,128,0,1,float16,float16,0,0.2571946581204732
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,96,96,128,0,1,float16,float16,0,0.702293316523234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,96,4,128,0,1,float16,float16,0,0.2609493335088094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,96,96,128,0,1,float16,float16,0,0.359935998916626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,96,8,128,0,1,float16,float16,0,0.26606933275858563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,96,1,128,0,1,float16,float16,0,0.13346133629480997
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,96,2,128,0,1,float16,float16,0,0.1358506679534912
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,96,4,128,0,1,float16,float16,0,0.13687466581662497
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,96,8,128,0,1,float16,float16,0,0.13823999961217245
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,96,1,128,0,1,float16,float16,0,0.07099733253320058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,96,96,128,0,1,float16,float16,0,0.10376532872517903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,96,2,128,0,1,float16,float16,0,0.07202133536338806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,96,96,128,0,1,float16,float16,0,0.19182932376861572
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,96,4,128,0,1,float16,float16,0,0.0718506673971812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,96,1,128,0,1,float16,float16,0,0.04113066693147024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,96,8,128,0,1,float16,float16,0,0.0730453332265218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,96,96,128,0,1,float16,float16,0,0.04795733094215393
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,96,4,128,0,1,float16,float16,0,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,96,2,128,0,1,float16,float16,0,0.04130133241415024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,96,8,128,0,1,float16,float16,0,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,96,1,128,0,1,float16,float16,0,0.02628266563018163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,96,2,128,0,1,float16,float16,0,0.0264533335963885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,96,4,128,0,1,float16,float16,0,0.0264533335963885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,96,8,128,0,1,float16,float16,0,0.0264533335963885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,96,96,128,0,1,float16,float16,0,0.0266239990790685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,96,1,128,0,1,float16,float16,0,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,96,96,128,0,1,float16,float16,0,0.018602666755517323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,96,2,128,0,1,float16,float16,0,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,96,4,128,0,1,float16,float16,0,0.018602666755517323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,96,8,128,0,1,float16,float16,0,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,96,1,128,0,1,float16,float16,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,96,96,128,0,1,float16,float16,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,96,4,128,0,1,float16,float16,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,96,8,128,0,1,float16,float16,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,96,2,128,0,1,float16,float16,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,96,1,128,0,1,float16,float16,0,0.38365864753723145
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,96,2,128,0,1,float16,float16,0,0.38946131865183514
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,16,96,1,128,0,1,float16,float16,0,0.7569066683451334
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,16,96,2,128,0,1,float16,float16,0,0.7649280230204264
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,16,96,4,128,0,1,float16,float16,0,0.7686826388041178
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,96,4,128,0,1,float16,float16,0,0.39082666238149005
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,16,96,8,128,0,1,float16,float16,0,0.7748266855875651
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,96,96,128,0,1,float16,float16,0,0.45636268456776935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,96,1,128,0,1,float16,float16,0,0.19899733861287436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,96,4,128,0,1,float16,float16,0,0.20292266209920248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,96,2,128,0,1,float16,float16,0,0.19950934251149496
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,96,8,128,0,1,float16,float16,0,0.2032639980316162
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,96,8,128,0,1,float16,float16,0,0.3933866818745931
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,96,96,128,0,1,float16,float16,0,0.12424533565839131
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,96,96,128,0,1,float16,float16,0,0.23552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,96,1,128,0,1,float16,float16,0,0.1013759970664978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,96,2,128,0,1,float16,float16,0,0.10222933689753215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,96,4,128,0,1,float16,float16,0,0.10274133086204529
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,96,8,128,0,1,float16,float16,0,0.10427733262379964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,96,1,128,0,1,float16,float16,0,0.056661332647005715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,96,2,128,0,1,float16,float16,0,0.057002668579419456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,96,4,128,0,1,float16,float16,0,0.05717333157857259
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,96,8,128,0,1,float16,float16,0,0.057002668579419456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,96,96,128,0,1,float16,float16,0,0.061610668897628784
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,96,96,128,0,1,float16,float16,0,0.03379199902216593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,96,1,128,0,1,float16,float16,0,0.0341333324710528
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,96,2,128,0,1,float16,float16,0,0.0341333324710528
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,96,4,128,0,1,float16,float16,0,0.034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,96,8,128,0,1,float16,float16,0,0.034815999368826546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,96,96,128,0,1,float16,float16,0,0.02218666672706604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,96,1,128,0,1,float16,float16,0,0.02218666672706604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,96,4,128,0,1,float16,float16,0,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,96,8,128,0,1,float16,float16,0,0.022357332209746044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,96,1,128,0,1,float16,float16,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,96,2,128,0,1,float16,float16,0,0.016554666062196095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,96,96,128,0,1,float16,float16,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,96,4,128,0,1,float16,float16,0,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,96,8,128,0,1,float16,float16,0,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,96,2,128,0,1,float16,float16,0,0.022357332209746044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,96,96,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,96,1,128,0,1,float16,float16,0,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,96,4,128,0,1,float16,float16,0,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,96,2,128,0,1,float16,float16,0,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,96,8,128,0,1,float16,float16,0,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,64,1,128,0,1,float16,float16,0,117.89926147460938
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,64,2,128,0,1,float16,float16,0,117.00206502278645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,64,4,128,0,1,float16,float16,0,116.18320719401042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,64,1,128,0,1,float16,float16,0,58.30604553222656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16384,64,1,128,0,1,float16,float16,0,237.2404988606771
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16384,64,2,128,0,1,float16,float16,0,233.61332194010416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,64,2,128,0,1,float16,float16,0,58.58252970377604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,64,8,128,0,1,float16,float16,0,117.99535115559895
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,64,64,128,0,1,float16,float16,0,59.56266784667969
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,64,64,128,0,1,float16,float16,0,119.6583251953125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16384,64,8,128,0,1,float16,float16,0,237.55126953125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16384,64,4,128,0,1,float16,float16,0,237.2776896158854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,64,1,128,0,1,float16,float16,0,30.102015177408855
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,64,4,128,0,1,float16,float16,0,59.95007832845052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,64,2,128,0,1,float16,float16,0,29.77655537923177
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,64,8,128,0,1,float16,float16,0,58.79500834147135
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,64,4,128,0,1,float16,float16,0,30.075050354003906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,64,8,128,0,1,float16,float16,0,30.181546529134113
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,64,64,128,0,1,float16,float16,0,29.786794026692707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,64,1,128,0,1,float16,float16,0,68.16853332519531
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,64,2,128,0,1,float16,float16,0,67.96543884277344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,64,4,128,0,1,float16,float16,0,66.65267435709636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,12288,64,2,128,0,1,float16,float16,0,134.6520792643229
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,64,1,128,0,1,float16,float16,0,33.54504648844401
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,12288,64,4,128,0,1,float16,float16,0,135.30179850260416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,12288,64,8,128,0,1,float16,float16,0,133.69429524739584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,64,8,128,0,1,float16,float16,0,68.09702555338542
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,12288,64,1,128,0,1,float16,float16,0,135.31647745768228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,64,64,128,0,1,float16,float16,0,34.489173889160156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,64,64,128,0,1,float16,float16,0,67.89853922526042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,64,2,128,0,1,float16,float16,0,33.88347625732422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,64,4,128,0,1,float16,float16,0,33.12981414794922
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,64,8,128,0,1,float16,float16,0,33.93245951334635
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,64,1,128,0,1,float16,float16,0,17.316864013671875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,64,2,128,0,1,float16,float16,0,17.32164255777995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,64,4,128,0,1,float16,float16,0,17.327957153320312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,64,8,128,0,1,float16,float16,0,17.065471649169922
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,64,64,128,0,1,float16,float16,0,17.202346801757812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,64,1,128,0,1,float16,float16,0,47.606953938802086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,64,2,128,0,1,float16,float16,0,46.638081868489586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,64,4,128,0,1,float16,float16,0,46.664703369140625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,10240,64,1,128,0,1,float16,float16,0,94.3436787923177
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,10240,64,2,128,0,1,float16,float16,0,93.5043436686198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,64,1,128,0,1,float16,float16,0,23.552683512369793
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,10240,64,4,128,0,1,float16,float16,0,95.33303833007812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,10240,64,8,128,0,1,float16,float16,0,95.19991048177083
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,64,2,128,0,1,float16,float16,0,23.6943359375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,64,8,128,0,1,float16,float16,0,47.70781962076823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,64,64,128,0,1,float16,float16,0,23.807828267415363
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,64,4,128,0,1,float16,float16,0,23.30248514811198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,64,1,128,0,1,float16,float16,0,12.221611022949219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,64,8,128,0,1,float16,float16,0,23.60302988688151
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,64,2,128,0,1,float16,float16,0,12.129280090332031
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,64,64,128,0,1,float16,float16,0,12.312576293945312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,64,64,128,0,1,float16,float16,0,48.3621571858724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,64,4,128,0,1,float16,float16,0,12.290218353271484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,64,8,128,0,1,float16,float16,0,11.96902338663737
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,64,2,128,0,1,float16,float16,0,62.104573567708336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,64,4,128,0,1,float16,float16,0,60.78532409667969
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,64,1,128,0,1,float16,float16,0,62.17369588216146
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,64,1,128,0,1,float16,float16,0,30.9749755859375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,8192,64,1,128,0,1,float16,float16,0,120.75110880533855
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,8192,64,2,128,0,1,float16,float16,0,123.05936686197917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,64,8,128,0,1,float16,float16,0,61.91070048014323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,8192,64,8,128,0,1,float16,float16,0,122.81463623046875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,64,2,128,0,1,float16,float16,0,30.671531677246094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,64,64,128,0,1,float16,float16,0,30.8677978515625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,8192,64,4,128,0,1,float16,float16,0,121.87613932291667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,64,4,128,0,1,float16,float16,0,30.347946166992188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,64,64,128,0,1,float16,float16,0,62.60923767089844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,64,8,128,0,1,float16,float16,0,30.684160868326824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,64,1,128,0,1,float16,float16,0,15.504725138346354
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,64,2,128,0,1,float16,float16,0,15.547562917073568
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,64,4,128,0,1,float16,float16,0,15.286954243977865
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,64,1,128,0,1,float16,float16,0,7.964501063028972
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,64,2,128,0,1,float16,float16,0,8.047615687052408
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,64,8,128,0,1,float16,float16,0,15.401983896891275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,64,4,128,0,1,float16,float16,0,8.002389272054037
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,64,8,128,0,1,float16,float16,0,8.041642506917318
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,64,64,128,0,1,float16,float16,0,7.848447799682617
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,64,64,128,0,1,float16,float16,0,15.779499053955078
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,64,1,128,0,1,float16,float16,0,35.267242431640625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,64,2,128,0,1,float16,float16,0,34.97130584716797
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,64,4,128,0,1,float16,float16,0,35.60038503011068
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,6144,64,1,128,0,1,float16,float16,0,69.88953653971355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,64,1,128,0,1,float16,float16,0,17.698645273844402
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,6144,64,2,128,0,1,float16,float16,0,71.42331949869792
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,6144,64,4,128,0,1,float16,float16,0,71.1533203125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,64,8,128,0,1,float16,float16,0,35.7027842203776
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,6144,64,8,128,0,1,float16,float16,0,70.52663675944011
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,64,64,128,0,1,float16,float16,0,17.791659037272137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,64,1,128,0,1,float16,float16,0,9.01751454671224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,64,2,128,0,1,float16,float16,0,17.465173085530598
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,64,4,128,0,1,float16,float16,0,17.744213104248047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,64,64,128,0,1,float16,float16,0,36.058624267578125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,64,8,128,0,1,float16,float16,0,17.83910369873047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,64,2,128,0,1,float16,float16,0,8.96785036722819
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,64,1,128,0,1,float16,float16,0,4.630186716715495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,64,4,128,0,1,float16,float16,0,8.71833610534668
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,64,2,128,0,1,float16,float16,0,4.580863952636719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,64,64,128,0,1,float16,float16,0,9.162410736083984
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,64,4,128,0,1,float16,float16,0,4.58513069152832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,64,8,128,0,1,float16,float16,0,4.548608144124349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,64,8,128,0,1,float16,float16,0,9.020586649576822
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,64,64,128,0,1,float16,float16,0,4.556970596313477
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,64,2,128,0,1,float16,float16,0,33.17350514729818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,64,1,128,0,1,float16,float16,0,32.479573567708336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,64,4,128,0,1,float16,float16,0,32.52326456705729
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,64,1,128,0,1,float16,float16,0,16.00102361043294
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,4096,64,2,128,0,1,float16,float16,0,65.2569580078125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,4096,64,1,128,0,1,float16,float16,0,66.2307840983073
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,4096,64,4,128,0,1,float16,float16,0,65.72288004557292
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,64,8,128,0,1,float16,float16,0,32.99259694417318
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,4096,64,8,128,0,1,float16,float16,0,66.20177205403645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,64,2,128,0,1,float16,float16,0,16.01058069864909
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,64,64,128,0,1,float16,float16,0,16.857770284016926
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,64,1,128,0,1,float16,float16,0,8.244906743367514
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,64,4,128,0,1,float16,float16,0,15.990613301595053
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,64,8,128,0,1,float16,float16,0,16.435882568359375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,64,2,128,0,1,float16,float16,0,8.272213617960611
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,64,64,128,0,1,float16,float16,0,34.13111368815104
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,64,4,128,0,1,float16,float16,0,8.085674921671549
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,64,1,128,0,1,float16,float16,0,4.080810546875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,64,64,128,0,1,float16,float16,0,8.623957316080729
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,64,2,128,0,1,float16,float16,0,4.174336115519206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,64,1,128,0,1,float16,float16,0,2.177194595336914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,64,4,128,0,1,float16,float16,0,4.151295979817708
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,64,8,128,0,1,float16,float16,0,4.178261439005534
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,64,8,128,0,1,float16,float16,0,8.196266810099283
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,64,64,128,0,1,float16,float16,0,4.261034647623698
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,64,64,128,0,1,float16,float16,0,2.194602648417155
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,64,2,128,0,1,float16,float16,0,2.187434673309326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,64,4,128,0,1,float16,float16,0,2.180095990498861
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,64,8,128,0,1,float16,float16,0,2.176682631174723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,64,1,128,0,1,float16,float16,0,19.106133778889973
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,64,2,128,0,1,float16,float16,0,19.327147165934246
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,64,4,128,0,1,float16,float16,0,18.97164789835612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,64,1,128,0,1,float16,float16,0,9.25593630472819
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,3072,64,1,128,0,1,float16,float16,0,38.677162170410156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,3072,64,2,128,0,1,float16,float16,0,38.999552408854164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,64,8,128,0,1,float16,float16,0,19.171839396158855
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,3072,64,8,128,0,1,float16,float16,0,38.89134979248047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,64,2,128,0,1,float16,float16,0,9.311914443969727
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,64,4,128,0,1,float16,float16,0,9.509717305501303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,3072,64,4,128,0,1,float16,float16,0,38.3092041015625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,64,64,128,0,1,float16,float16,0,10.117290496826172
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,64,1,128,0,1,float16,float16,0,4.812629381815593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,64,64,128,0,1,float16,float16,0,19.898197174072266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,64,2,128,0,1,float16,float16,0,4.699647903442383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,64,8,128,0,1,float16,float16,0,9.631231943766275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,64,1,128,0,1,float16,float16,0,2.4268800417582193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,64,4,128,0,1,float16,float16,0,4.807680130004883
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,64,4,128,0,1,float16,float16,0,2.4362667401631675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,64,8,128,0,1,float16,float16,0,4.794026692708333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,64,2,128,0,1,float16,float16,0,2.4388267199198403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,64,1,128,0,1,float16,float16,0,1.330858627955119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,64,64,128,0,1,float16,float16,0,2.5105066299438477
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,64,8,128,0,1,float16,float16,0,2.440021355946859
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,64,64,128,0,1,float16,float16,0,5.0414934158325195
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,64,2,128,0,1,float16,float16,0,1.3219839731852214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,64,64,128,0,1,float16,float16,0,1.3327360153198242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,64,4,128,0,1,float16,float16,0,1.3192533651987712
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,64,8,128,0,1,float16,float16,0,1.325055996576945
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,64,1,128,0,1,float16,float16,0,18.67315165201823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,64,2,128,0,1,float16,float16,0,18.54583485921224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,64,4,128,0,1,float16,float16,0,18.386603037516277
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,64,1,128,0,1,float16,float16,0,8.977578481038412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,2048,64,2,128,0,1,float16,float16,0,37.54990895589193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,2048,64,1,128,0,1,float16,float16,0,37.23861440022787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,2048,64,4,128,0,1,float16,float16,0,37.456382751464844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,64,8,128,0,1,float16,float16,0,18.81173324584961
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,64,2,128,0,1,float16,float16,0,8.993109385172525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,2048,64,8,128,0,1,float16,float16,0,37.67944590250651
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,64,1,128,0,1,float16,float16,0,4.503210703531901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,64,4,128,0,1,float16,float16,0,9.196032206217447
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,64,8,128,0,1,float16,float16,0,9.263274510701498
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,64,64,128,0,1,float16,float16,0,9.861973444620768
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,64,2,128,0,1,float16,float16,0,4.459349314371745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,64,64,128,0,1,float16,float16,0,19.500202178955078
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,64,4,128,0,1,float16,float16,0,4.539221445719401
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,64,1,128,0,1,float16,float16,0,2.2860800425211587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,64,2,128,0,1,float16,float16,0,2.2843732833862305
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,64,8,128,0,1,float16,float16,0,4.5619198481241865
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,64,64,128,0,1,float16,float16,0,4.957013448079427
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,64,1,128,0,1,float16,float16,0,1.2076373100280762
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,64,8,128,0,1,float16,float16,0,2.297002633412679
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,64,4,128,0,1,float16,float16,0,1.2064426740010579
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,64,2,128,0,1,float16,float16,0,1.2134400208791096
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,64,4,128,0,1,float16,float16,0,2.2860800425211587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,64,64,128,0,1,float16,float16,0,2.4378026326497397
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,64,1,128,0,1,float16,float16,0,0.6737919648488363
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,64,8,128,0,1,float16,float16,0,1.2178773085276287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,64,2,128,0,1,float16,float16,0,0.6744746367136637
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,64,64,128,0,1,float16,float16,0,0.6853973070780436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,64,4,128,0,1,float16,float16,0,0.6749866803487142
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,64,64,128,0,1,float16,float16,0,1.2622506618499756
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,64,8,128,0,1,float16,float16,0,0.6722559928894043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,64,1,128,0,1,float16,float16,0,11.076095581054688
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,64,2,128,0,1,float16,float16,0,11.209898630777994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,64,4,128,0,1,float16,float16,0,11.115008036295572
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,64,1,128,0,1,float16,float16,0,5.342720031738281
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1536,64,1,128,0,1,float16,float16,0,22.548309326171875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1536,64,2,128,0,1,float16,float16,0,22.326955159505207
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1536,64,4,128,0,1,float16,float16,0,22.74542999267578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1536,64,8,128,0,1,float16,float16,0,22.880767822265625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,64,8,128,0,1,float16,float16,0,11.243690490722656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,64,2,128,0,1,float16,float16,0,5.356202443440755
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,64,64,128,0,1,float16,float16,0,6.053717295328776
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,64,1,128,0,1,float16,float16,0,2.704554557800293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,64,2,128,0,1,float16,float16,0,2.701141357421875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,64,4,128,0,1,float16,float16,0,5.498538970947266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,64,8,128,0,1,float16,float16,0,5.402794520060222
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,64,64,128,0,1,float16,float16,0,12.26052220662435
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,64,4,128,0,1,float16,float16,0,2.700629234313965
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,64,1,128,0,1,float16,float16,0,1.3957120577494304
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,64,2,128,0,1,float16,float16,0,1.4044159253438313
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,64,4,128,0,1,float16,float16,0,1.4032212893168132
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,64,1,128,0,1,float16,float16,0,0.7504213651021322
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,64,8,128,0,1,float16,float16,0,2.734421412150065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,64,8,128,0,1,float16,float16,0,1.413119951883952
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,64,2,128,0,1,float16,float16,0,0.7524693012237549
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,64,64,128,0,1,float16,float16,0,3.011413256327311
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,64,64,128,0,1,float16,float16,0,1.4989652633666992
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,64,1,128,0,1,float16,float16,0,0.4288853406906128
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,64,4,128,0,1,float16,float16,0,0.7522986729939779
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,64,2,128,0,1,float16,float16,0,0.4317866563796997
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,64,64,128,0,1,float16,float16,0,0.7951359748840332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,64,4,128,0,1,float16,float16,0,0.43212799231211346
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,64,8,128,0,1,float16,float16,0,0.42905600865681964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,64,8,128,0,1,float16,float16,0,0.7545173168182373
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,64,64,128,0,1,float16,float16,0,0.4389546712239583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,64,1,128,0,1,float16,float16,0,10.945365905761719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,64,2,128,0,1,float16,float16,0,10.80251693725586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,64,4,128,0,1,float16,float16,0,10.939221700032553
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,64,1,128,0,1,float16,float16,0,5.392896016438802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,1024,64,1,128,0,1,float16,float16,0,22.083412170410156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,1024,64,2,128,0,1,float16,float16,0,21.67534891764323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,64,8,128,0,1,float16,float16,0,11.093674977620443
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,64,2,128,0,1,float16,float16,0,5.419349034627278
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,1024,64,4,128,0,1,float16,float16,0,22.190762837727863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,1024,64,8,128,0,1,float16,float16,0,22.128982543945312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,64,64,128,0,1,float16,float16,0,6.34231440226237
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,64,4,128,0,1,float16,float16,0,5.442218780517578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,64,8,128,0,1,float16,float16,0,5.397504170735677
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,64,1,128,0,1,float16,float16,0,2.6757119496663413
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,64,2,128,0,1,float16,float16,0,2.675029436747233
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,64,4,128,0,1,float16,float16,0,2.6745173136393228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,64,64,128,0,1,float16,float16,0,12.819114685058594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,64,8,128,0,1,float16,float16,0,2.7028481165568032
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,64,1,128,0,1,float16,float16,0,1.3501440684000652
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,64,2,128,0,1,float16,float16,0,1.357141335805257
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,64,4,128,0,1,float16,float16,0,1.3530453046162922
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,64,64,128,0,1,float16,float16,0,3.178154627482096
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,64,1,128,0,1,float16,float16,0,0.6930773258209229
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,64,2,128,0,1,float16,float16,0,0.6971733570098877
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,64,8,128,0,1,float16,float16,0,1.3644800186157227
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,64,4,128,0,1,float16,float16,0,0.7010986804962158
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,64,8,128,0,1,float16,float16,0,0.7051946322123209
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,64,1,128,0,1,float16,float16,0,0.3696639935175578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,64,64,128,0,1,float16,float16,0,0.8093012968699137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,64,64,128,0,1,float16,float16,0,0.4230826695760091
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,64,2,128,0,1,float16,float16,0,0.370688001314799
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,64,64,128,0,1,float16,float16,0,1.5802027384440105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,64,8,128,0,1,float16,float16,0,0.37444265683492023
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,64,4,128,0,1,float16,float16,0,0.3691519896189372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,64,1,128,0,1,float16,float16,0,0.20258132616678873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,64,64,128,0,1,float16,float16,0,0.2300586700439453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,64,2,128,0,1,float16,float16,0,0.2032639980316162
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,64,4,128,0,1,float16,float16,0,0.2032639980316162
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,64,8,128,0,1,float16,float16,0,0.20616533358891806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,64,1,128,0,1,float16,float16,0,7.281322479248047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,64,2,128,0,1,float16,float16,0,7.386112213134766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,64,4,128,0,1,float16,float16,0,7.2695465087890625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,64,1,128,0,1,float16,float16,0,3.601578712463379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,512,64,2,128,0,1,float16,float16,0,14.865578969319662
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,512,64,4,128,0,1,float16,float16,0,14.866432189941406
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,64,2,128,0,1,float16,float16,0,3.6097707748413086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,64,8,128,0,1,float16,float16,0,7.521792093912761
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,512,64,1,128,0,1,float16,float16,0,14.926165262858072
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,64,64,128,0,1,float16,float16,0,4.608682632446289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,512,64,8,128,0,1,float16,float16,0,15.20196278889974
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,64,1,128,0,1,float16,float16,0,1.8119680086771648
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,64,4,128,0,1,float16,float16,0,3.6439040501912436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,64,2,128,0,1,float16,float16,0,1.8182826042175293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,64,4,128,0,1,float16,float16,0,1.8201600710550945
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,64,8,128,0,1,float16,float16,0,3.6904961268107095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,64,8,128,0,1,float16,float16,0,1.8445653915405273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,64,64,128,0,1,float16,float16,0,9.34997304280599
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,64,1,128,0,1,float16,float16,0,0.92194135983785
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,64,2,128,0,1,float16,float16,0,0.917674700419108
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,64,4,128,0,1,float16,float16,0,0.9183573722839355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,64,1,128,0,1,float16,float16,0,0.48742401599884033
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,64,2,128,0,1,float16,float16,0,0.4742826620737712
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,64,64,128,0,1,float16,float16,0,2.3045120239257812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,64,4,128,0,1,float16,float16,0,0.47411199410756427
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,64,8,128,0,1,float16,float16,0,0.9359359741210938
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,64,64,128,0,1,float16,float16,0,0.5811200141906738
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,64,8,128,0,1,float16,float16,0,0.48230401674906415
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,64,64,128,0,1,float16,float16,0,1.1352746486663818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,64,1,128,0,1,float16,float16,0,0.2510506709416707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,64,2,128,0,1,float16,float16,0,0.2532693346341451
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,64,64,128,0,1,float16,float16,0,0.3037866751352946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,64,1,128,0,1,float16,float16,0,0.1372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,64,4,128,0,1,float16,float16,0,0.2515626748402913
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,64,2,128,0,1,float16,float16,0,0.13738666971524557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,64,64,128,0,1,float16,float16,0,0.16622933745384216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,64,4,128,0,1,float16,float16,0,0.13875200351079306
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,64,8,128,0,1,float16,float16,0,0.25497599442799884
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,64,1,128,0,1,float16,float16,0,0.0795306662718455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,64,2,128,0,1,float16,float16,0,0.07850666840871175
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,64,8,128,0,1,float16,float16,0,0.1397760013739268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,64,64,128,0,1,float16,float16,0,0.09301333626111348
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,64,8,128,0,1,float16,float16,0,0.07884799937407176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,64,4,128,0,1,float16,float16,0,0.07918933530648549
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,64,1,128,0,1,float16,float16,0,2.739199956258138
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,64,2,128,0,1,float16,float16,0,2.745855967203776
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,64,4,128,0,1,float16,float16,0,2.757290522257487
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,64,1,128,0,1,float16,float16,0,1.3801813125610352
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,256,64,1,128,0,1,float16,float16,0,5.541376113891602
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,256,64,2,128,0,1,float16,float16,0,5.549909591674805
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,256,64,4,128,0,1,float16,float16,0,5.594624201456706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,64,8,128,0,1,float16,float16,0,2.849280039469401
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,256,64,8,128,0,1,float16,float16,0,5.749930699666341
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,64,2,128,0,1,float16,float16,0,1.377621332804362
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,64,4,128,0,1,float16,float16,0,1.3941760063171387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,64,64,128,0,1,float16,float16,0,1.8974720637003581
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,64,1,128,0,1,float16,float16,0,0.7038293679555258
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,64,2,128,0,1,float16,float16,0,0.7028053601582845
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,64,8,128,0,1,float16,float16,0,1.4160213470458984
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,64,4,128,0,1,float16,float16,0,0.7014400164286295
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,64,64,128,0,1,float16,float16,0,3.787775993347168
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,64,1,128,0,1,float16,float16,0,0.35891199111938477
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,64,64,128,0,1,float16,float16,0,0.9772373040517172
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,64,2,128,0,1,float16,float16,0,0.359935998916626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,64,8,128,0,1,float16,float16,0,0.7138986587524414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,64,8,128,0,1,float16,float16,0,0.3676160176595052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,64,1,128,0,1,float16,float16,0,0.18961066007614136
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,64,2,128,0,1,float16,float16,0,0.19182932376861572
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,64,4,128,0,1,float16,float16,0,0.36266668637593585
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,64,64,128,0,1,float16,float16,0,0.5009066661198934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,64,64,128,0,1,float16,float16,0,0.26265599330266315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,64,4,128,0,1,float16,float16,0,0.19165867567062378
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,64,8,128,0,1,float16,float16,0,0.19524266322453818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,64,1,128,0,1,float16,float16,0,0.10376532872517903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,64,2,128,0,1,float16,float16,0,0.10359467069307964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,64,8,128,0,1,float16,float16,0,0.10615467031796773
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,64,64,128,0,1,float16,float16,0,0.13687466581662497
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,64,64,128,0,1,float16,float16,0,0.07372800012429555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,64,1,128,0,1,float16,float16,0,0.0602453351020813
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,64,4,128,0,1,float16,float16,0,0.10513066252072652
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,64,4,128,0,1,float16,float16,0,0.06007466713587443
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,64,2,128,0,1,float16,float16,0,0.05973333120346069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,64,8,128,0,1,float16,float16,0,0.0602453351020813
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,64,2,128,0,1,float16,float16,0,0.03942399968703588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,64,64,128,0,1,float16,float16,0,0.04215466479460398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,64,4,128,0,1,float16,float16,0,0.03942399968703588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,64,1,128,0,1,float16,float16,0,0.040106666584809623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,64,8,128,0,1,float16,float16,0,0.03942399968703588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,64,1,128,0,1,float16,float16,0,1.1805013020833333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,64,4,128,0,1,float16,float16,0,1.188693364461263
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,64,2,128,0,1,float16,float16,0,1.1852800051371257
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,128,64,1,128,0,1,float16,float16,0,2.3522987365722656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,128,64,2,128,0,1,float16,float16,0,2.369365374247233
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,128,64,4,128,0,1,float16,float16,0,2.378239949544271
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,64,1,128,0,1,float16,float16,0,0.59989333152771
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,128,64,8,128,0,1,float16,float16,0,2.453674634297689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,64,2,128,0,1,float16,float16,0,0.6056960026423136
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,64,8,128,0,1,float16,float16,0,1.215488036473592
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,64,4,128,0,1,float16,float16,0,0.6091093222300211
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,64,64,128,0,1,float16,float16,0,1.7974613507588704
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,64,1,128,0,1,float16,float16,0,0.3094186584154765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,64,8,128,0,1,float16,float16,0,0.6224213441212972
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,64,2,128,0,1,float16,float16,0,0.30822400252024335
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,64,64,128,0,1,float16,float16,0,0.9113600254058838
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,64,4,128,0,1,float16,float16,0,0.31214932600657147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,64,1,128,0,1,float16,float16,0,0.183296004931132
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,64,8,128,0,1,float16,float16,0,0.318122665087382
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,64,64,128,0,1,float16,float16,0,0.4681386550267537
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,64,2,128,0,1,float16,float16,0,0.16264533003171286
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,64,64,128,0,1,float16,float16,0,0.24951465924580893
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,64,4,128,0,1,float16,float16,0,0.1629866659641266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,64,64,128,0,1,float16,float16,0,0.12868266304334006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,64,8,128,0,1,float16,float16,0,0.16639999548594156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,64,1,128,0,1,float16,float16,0,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,64,2,128,0,1,float16,float16,0,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,64,8,128,0,1,float16,float16,0,0.08891733487447102
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,64,64,128,0,1,float16,float16,0,0.06502399841944377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,64,1,128,0,1,float16,float16,0,0.048810665806134544
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,64,4,128,0,1,float16,float16,0,0.08703999718030293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,64,2,128,0,1,float16,float16,0,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,64,8,128,0,1,float16,float16,0,0.049322664737701416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,64,1,128,0,1,float16,float16,0,0.0339626669883728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,64,64,128,0,1,float16,float16,0,0.03618133316437403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,64,4,128,0,1,float16,float16,0,0.049322664737701416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,64,2,128,0,1,float16,float16,0,0.03328000009059906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,64,4,128,0,1,float16,float16,0,0.03345066557327906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,64,64,128,0,1,float16,float16,0,0.02611200014750163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,64,1,128,0,1,float16,float16,0,0.025429333249727886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,64,2,128,0,1,float16,float16,0,0.024746666351954143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,64,8,128,0,1,float16,float16,0,0.03345066557327906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,64,4,128,0,1,float16,float16,0,0.025087999800841015
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,64,8,128,0,1,float16,float16,0,0.02491733431816101
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,64,2,128,0,1,float16,float16,0,0.6036479870478312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,64,1,128,0,1,float16,float16,0,0.579584002494812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,64,64,1,128,0,1,float16,float16,0,1.1398826440175374
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,64,64,2,128,0,1,float16,float16,0,1.1890347003936768
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,64,4,128,0,1,float16,float16,0,0.6133759816487631
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,64,64,4,128,0,1,float16,float16,0,1.2107093334197998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,64,64,8,128,0,1,float16,float16,0,1.2521813710530598
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,64,1,128,0,1,float16,float16,0,0.30719999472300213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,64,2,128,0,1,float16,float16,0,0.3155626654624939
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,64,64,128,0,1,float16,float16,0,0.9115306536356608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,64,4,128,0,1,float16,float16,0,0.3155626654624939
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,64,8,128,0,1,float16,float16,0,0.3275093237559001
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,64,8,128,0,1,float16,float16,0,0.6324906746546427
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,64,64,128,0,1,float16,float16,0,0.4695039987564087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,64,1,128,0,1,float16,float16,0,0.1599146624406179
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,64,2,128,0,1,float16,float16,0,0.1629866659641266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,64,4,128,0,1,float16,float16,0,0.16725333531697592
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,64,64,128,0,1,float16,float16,0,0.24985599517822266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,64,8,128,0,1,float16,float16,0,0.17169066270192465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,64,1,128,0,1,float16,float16,0,0.08533333738644917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,64,2,128,0,1,float16,float16,0,0.08584533135096233
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,64,64,128,0,1,float16,float16,0,0.12902399897575378
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,64,4,128,0,1,float16,float16,0,0.08703999718030293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,64,8,128,0,1,float16,float16,0,0.08959999680519104
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,64,1,128,0,1,float16,float16,0,0.048810665806134544
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,64,64,128,0,1,float16,float16,0,0.06553600231806438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,64,1,128,0,1,float16,float16,0,0.034645333886146545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,64,8,128,0,1,float16,float16,0,0.049322664737701416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,64,4,128,0,1,float16,float16,0,0.04898133377234141
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,64,64,128,0,1,float16,float16,0,0.036864000062147774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,64,2,128,0,1,float16,float16,0,0.07765333354473114
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,64,2,128,0,1,float16,float16,0,0.034304000437259674
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,64,4,128,0,1,float16,float16,0,0.0339626669883728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,64,8,128,0,1,float16,float16,0,0.0341333324710528
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,64,64,128,0,1,float16,float16,0,0.02491733431816101
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,64,1,128,0,1,float16,float16,0,0.024234667420387268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,64,4,128,0,1,float16,float16,0,0.024234667420387268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,64,1,128,0,1,float16,float16,0,0.018602666755517323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,64,64,128,0,1,float16,float16,0,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,64,2,128,0,1,float16,float16,0,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,64,2,128,0,1,float16,float16,0,0.0240639994541804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,64,8,128,0,1,float16,float16,0,0.024405332903067272
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,64,4,128,0,1,float16,float16,0,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,64,8,128,0,1,float16,float16,0,0.018432000031073887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,64,2,128,0,1,float16,float16,0,0.3418453137079875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,32,64,1,128,0,1,float16,float16,0,0.6647466818491617
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,64,4,128,0,1,float16,float16,0,0.3432106574376424
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,32,64,2,128,0,1,float16,float16,0,0.6698666413625082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,64,1,128,0,1,float16,float16,0,0.3397973378499349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,32,64,4,128,0,1,float16,float16,0,0.6702079772949219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,32,64,8,128,0,1,float16,float16,0,0.6997333367665609
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,64,64,128,0,1,float16,float16,0,0.47308798631032306
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,64,1,128,0,1,float16,float16,0,0.17681066195170084
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,64,2,128,0,1,float16,float16,0,0.1790293256441752
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,64,8,128,0,1,float16,float16,0,0.34918399651845294
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,64,8,128,0,1,float16,float16,0,0.18414932489395142
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,64,64,128,0,1,float16,float16,0,0.13124266266822815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,64,64,128,0,1,float16,float16,0,0.2486613392829895
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,64,4,128,0,1,float16,float16,0,0.17834667364756265
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,64,2,128,0,1,float16,float16,0,0.09352533022562663
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,64,4,128,0,1,float16,float16,0,0.0942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,64,1,128,0,1,float16,float16,0,0.09318400422732036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,64,8,128,0,1,float16,float16,0,0.09727999567985535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,64,64,128,0,1,float16,float16,0,0.0653653343518575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,64,1,128,0,1,float16,float16,0,0.0506879985332489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,64,2,128,0,1,float16,float16,0,0.05034666756788889
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,64,8,128,0,1,float16,float16,0,0.0506879985332489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,64,4,128,0,1,float16,float16,0,0.04983466863632202
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,64,64,128,0,1,float16,float16,0,0.03515733281771342
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,64,2,128,0,1,float16,float16,0,0.03328000009059906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,64,1,128,0,1,float16,float16,0,0.03328000009059906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,64,4,128,0,1,float16,float16,0,0.03345066557327906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,64,8,128,0,1,float16,float16,0,0.03328000009059906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,64,64,128,0,1,float16,float16,0,0.025600001215934753
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,64,1,128,0,1,float16,float16,0,0.025087999800841015
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,64,4,128,0,1,float16,float16,0,0.025258667767047882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,64,8,128,0,1,float16,float16,0,0.025258667767047882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,64,1,128,0,1,float16,float16,0,0.017407999684413273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,64,64,128,0,1,float16,float16,0,0.0170666662355264
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,64,2,128,0,1,float16,float16,0,0.0170666662355264
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,64,4,128,0,1,float16,float16,0,0.0170666662355264
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,64,2,128,0,1,float16,float16,0,0.025429333249727886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,64,8,128,0,1,float16,float16,0,0.017237332959969837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,64,64,128,0,1,float16,float16,0,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,64,1,128,0,1,float16,float16,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,64,2,128,0,1,float16,float16,0,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,64,8,128,0,1,float16,float16,0,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,64,4,128,0,1,float16,float16,0,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,16,64,1,128,0,1,float16,float16,0,0.5237760146458944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,64,1,128,0,1,float16,float16,0,0.26282666126887005
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,16,64,4,128,0,1,float16,float16,0,0.5222400029500326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,64,2,128,0,1,float16,float16,0,0.26521599292755127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,16,64,2,128,0,1,float16,float16,0,0.5196799834569296
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,16,64,8,128,0,1,float16,float16,0,0.5294080177942911
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,64,4,128,0,1,float16,float16,0,0.26692267258961994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,64,64,128,0,1,float16,float16,0,0.31061333417892456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,64,1,128,0,1,float16,float16,0,0.13294933239618936
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,64,2,128,0,1,float16,float16,0,0.13363200426101685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,64,8,128,0,1,float16,float16,0,0.27084799607594806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,64,8,128,0,1,float16,float16,0,0.13806933164596558
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,64,64,128,0,1,float16,float16,0,0.16059733430544534
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,64,1,128,0,1,float16,float16,0,0.07082666456699371
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,64,64,128,0,1,float16,float16,0,0.08260266482830048
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,64,4,128,0,1,float16,float16,0,0.13482667009035745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,64,2,128,0,1,float16,float16,0,0.07099733253320058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,64,4,128,0,1,float16,float16,0,0.07099733253320058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,64,1,128,0,1,float16,float16,0,0.04181333382924398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,64,8,128,0,1,float16,float16,0,0.07133866846561432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,64,2,128,0,1,float16,float16,0,0.04181333382924398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,64,64,128,0,1,float16,float16,0,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,64,4,128,0,1,float16,float16,0,0.041984001795450844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,64,1,128,0,1,float16,float16,0,0.02867199977238973
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,64,64,128,0,1,float16,float16,0,0.02867199977238973
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,64,2,128,0,1,float16,float16,0,0.0290133332212766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,64,8,128,0,1,float16,float16,0,0.029525332152843475
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,64,4,128,0,1,float16,float16,0,0.02918400118748347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,64,64,128,0,1,float16,float16,0,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,64,8,128,0,1,float16,float16,0,0.04215466479460398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,64,1,128,0,1,float16,float16,0,0.020992000897725422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,64,2,128,0,1,float16,float16,0,0.020309332758188248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,64,4,128,0,1,float16,float16,0,0.020821332931518555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,64,8,128,0,1,float16,float16,0,0.020309332758188248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,64,2,128,0,1,float16,float16,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,64,1,128,0,1,float16,float16,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,64,64,128,0,1,float16,float16,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,64,8,128,0,1,float16,float16,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,64,4,128,0,1,float16,float16,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,64,64,128,0,1,float16,float16,0,0.013823999712864557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,64,1,128,0,1,float16,float16,0,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,64,2,128,0,1,float16,float16,0,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,64,4,128,0,1,float16,float16,0,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,64,8,128,0,1,float16,float16,0,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,48,1,128,0,1,float16,float16,0,89.82015991210938
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,48,2,128,0,1,float16,float16,0,87.75372314453125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,48,4,128,0,1,float16,float16,0,89.8324483235677
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,48,1,128,0,1,float16,float16,0,44.50713602701823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16384,48,1,128,0,1,float16,float16,0,175.61566162109375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16384,48,4,128,0,1,float16,float16,0,175.3178507486979
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16384,48,2,128,0,1,float16,float16,0,176.9569295247396
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,48,48,128,0,1,float16,float16,0,44.03677876790365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,48,48,128,0,1,float16,float16,0,89.798828125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,48,2,128,0,1,float16,float16,0,44.497406005859375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,48,8,128,0,1,float16,float16,0,89.63209025065105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16384,48,8,128,0,1,float16,float16,0,178.01898193359375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,48,1,128,0,1,float16,float16,0,22.159019470214844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,48,2,128,0,1,float16,float16,0,22.669825236002605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,48,4,128,0,1,float16,float16,0,22.67511494954427
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,48,4,128,0,1,float16,float16,0,44.078592936197914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,48,8,128,0,1,float16,float16,0,22.606165568033855
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,48,48,128,0,1,float16,float16,0,22.828203837076824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,48,8,128,0,1,float16,float16,0,44.03985087076823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,48,1,128,0,1,float16,float16,0,50.990763346354164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,48,2,128,0,1,float16,float16,0,50.45504252115885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,48,4,128,0,1,float16,float16,0,49.926483154296875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,12288,48,1,128,0,1,float16,float16,0,101.65453084309895
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,12288,48,2,128,0,1,float16,float16,0,99.47170003255208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,48,1,128,0,1,float16,float16,0,25.207125345865887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,12288,48,4,128,0,1,float16,float16,0,101.6265360514323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,48,2,128,0,1,float16,float16,0,25.531903584798176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,12288,48,8,128,0,1,float16,float16,0,101.3896484375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,48,48,128,0,1,float16,float16,0,50.980865478515625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,48,8,128,0,1,float16,float16,0,50.98700968424479
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,48,1,128,0,1,float16,float16,0,13.082623799641928
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,48,4,128,0,1,float16,float16,0,24.979286193847656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,48,48,128,0,1,float16,float16,0,25.739776611328125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,48,8,128,0,1,float16,float16,0,25.108479817708332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,48,2,128,0,1,float16,float16,0,13.040128072102865
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,48,4,128,0,1,float16,float16,0,12.790101369222006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,48,48,128,0,1,float16,float16,0,13.141674041748047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,48,8,128,0,1,float16,float16,0,13.063680013020834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,48,1,128,0,1,float16,float16,0,35.04554748535156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,48,2,128,0,1,float16,float16,0,35.46282704671224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,48,4,128,0,1,float16,float16,0,35.79460144042969
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,10240,48,1,128,0,1,float16,float16,0,71.65934753417969
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,48,1,128,0,1,float16,float16,0,17.936725616455078
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,10240,48,2,128,0,1,float16,float16,0,70.26397705078125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,10240,48,4,128,0,1,float16,float16,0,70.73365275065105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,48,2,128,0,1,float16,float16,0,17.94815953572591
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,10240,48,8,128,0,1,float16,float16,0,70.53448486328125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,48,8,128,0,1,float16,float16,0,35.100842793782554
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,48,4,128,0,1,float16,float16,0,18.009429931640625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,48,48,128,0,1,float16,float16,0,18.172757466634113
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,48,48,128,0,1,float16,float16,0,36.16392517089844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,48,1,128,0,1,float16,float16,0,9.156949361165365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,48,8,128,0,1,float16,float16,0,17.548800150553387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,48,2,128,0,1,float16,float16,0,9.057109196980795
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,48,4,128,0,1,float16,float16,0,9.093631744384766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,48,8,128,0,1,float16,float16,0,9.104213078816732
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,48,48,128,0,1,float16,float16,0,9.210197448730469
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,48,1,128,0,1,float16,float16,0,45.53301493326823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,48,2,128,0,1,float16,float16,0,46.39300028483073
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,48,4,128,0,1,float16,float16,0,45.91649881998698
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,48,1,128,0,1,float16,float16,0,23.122601826985676
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,8192,48,1,128,0,1,float16,float16,0,92.5673828125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,8192,48,2,128,0,1,float16,float16,0,92.37537638346355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,8192,48,4,128,0,1,float16,float16,0,92.89898681640625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,48,8,128,0,1,float16,float16,0,45.524993896484375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,48,2,128,0,1,float16,float16,0,23.051434834798176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,8192,48,8,128,0,1,float16,float16,0,91.02813720703125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,48,48,128,0,1,float16,float16,0,23.682047526041668
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,48,48,128,0,1,float16,float16,0,46.2008310953776
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,48,4,128,0,1,float16,float16,0,23.223467508951824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,48,1,128,0,1,float16,float16,0,11.573930104573568
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,48,4,128,0,1,float16,float16,0,11.412651062011719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,48,2,128,0,1,float16,float16,0,11.619327545166016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,48,1,128,0,1,float16,float16,0,5.963605244954427
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,48,8,128,0,1,float16,float16,0,11.72121556599935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,48,48,128,0,1,float16,float16,0,11.848704020182291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,48,2,128,0,1,float16,float16,0,5.932885487874349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,48,8,128,0,1,float16,float16,0,23.04614512125651
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,48,48,128,0,1,float16,float16,0,5.960874557495117
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,48,4,128,0,1,float16,float16,0,5.8753706614176435
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,48,8,128,0,1,float16,float16,0,5.98579216003418
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,48,1,128,0,1,float16,float16,0,26.1034673055013
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,48,2,128,0,1,float16,float16,0,25.954986572265625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,48,4,128,0,1,float16,float16,0,26.51477305094401
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,48,1,128,0,1,float16,float16,0,13.166250864664713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,6144,48,1,128,0,1,float16,float16,0,53.35791015625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,6144,48,2,128,0,1,float16,float16,0,53.41662089029948
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,6144,48,4,128,0,1,float16,float16,0,53.37651062011719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,6144,48,8,128,0,1,float16,float16,0,52.78668721516927
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,48,8,128,0,1,float16,float16,0,26.24085235595703
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,48,2,128,0,1,float16,float16,0,13.157034556070963
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,48,4,128,0,1,float16,float16,0,13.360298156738281
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,48,48,128,0,1,float16,float16,0,13.40398915608724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,48,48,128,0,1,float16,float16,0,27.392171223958332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,48,1,128,0,1,float16,float16,0,6.653098424275716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,48,8,128,0,1,float16,float16,0,13.355519612630209
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,48,48,128,0,1,float16,float16,0,6.901930491129558
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,48,2,128,0,1,float16,float16,0,6.62391471862793
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,48,1,128,0,1,float16,float16,0,3.466069221496582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,48,4,128,0,1,float16,float16,0,6.703957239786784
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,48,2,128,0,1,float16,float16,0,3.4382505416870117
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,48,8,128,0,1,float16,float16,0,6.766762415568034
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,48,4,128,0,1,float16,float16,0,3.4571946461995444
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,48,48,128,0,1,float16,float16,0,3.452586809794108
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,48,8,128,0,1,float16,float16,0,3.431253433227539
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,48,1,128,0,1,float16,float16,0,24.663551330566406
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,48,2,128,0,1,float16,float16,0,24.27972157796224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,48,4,128,0,1,float16,float16,0,24.72789255777995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,48,1,128,0,1,float16,float16,0,12.28561019897461
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,4096,48,1,128,0,1,float16,float16,0,48.59733581542969
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,4096,48,4,128,0,1,float16,float16,0,49.22845967610677
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,4096,48,2,128,0,1,float16,float16,0,49.480194091796875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,48,2,128,0,1,float16,float16,0,12.304725646972656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,48,8,128,0,1,float16,float16,0,24.8089599609375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,48,4,128,0,1,float16,float16,0,12.109141031901041
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,48,48,128,0,1,float16,float16,0,12.507818857828775
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,4096,48,8,128,0,1,float16,float16,0,49.13800557454427
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,48,1,128,0,1,float16,float16,0,6.148096084594727
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,48,48,128,0,1,float16,float16,0,25.65734354654948
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,48,2,128,0,1,float16,float16,0,6.210730870564778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,48,4,128,0,1,float16,float16,0,6.073173522949219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,48,1,128,0,1,float16,float16,0,3.1127894719441733
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,48,2,128,0,1,float16,float16,0,3.1092052459716797
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,48,8,128,0,1,float16,float16,0,12.281684875488281
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,48,4,128,0,1,float16,float16,0,3.1133012771606445
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,48,48,128,0,1,float16,float16,0,6.4460798899332685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,48,8,128,0,1,float16,float16,0,6.039722442626953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,48,1,128,0,1,float16,float16,0,1.664511998494466
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,48,2,128,0,1,float16,float16,0,1.666218598683675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,48,48,128,0,1,float16,float16,0,3.174229303995768
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,48,4,128,0,1,float16,float16,0,1.6585386594136555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,48,8,128,0,1,float16,float16,0,3.0977706909179688
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,48,48,128,0,1,float16,float16,0,1.6723626454671223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,48,8,128,0,1,float16,float16,0,1.6634880701700847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,48,1,128,0,1,float16,float16,0,14.258346557617188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,48,2,128,0,1,float16,float16,0,14.572884877522787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,48,4,128,0,1,float16,float16,0,14.531754811604818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,48,1,128,0,1,float16,float16,0,7.207594553629558
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,3072,48,1,128,0,1,float16,float16,0,28.346710205078125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,3072,48,2,128,0,1,float16,float16,0,29.206186930338543
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,3072,48,4,128,0,1,float16,float16,0,28.961110432942707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,48,8,128,0,1,float16,float16,0,14.276949564615885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,48,48,128,0,1,float16,float16,0,7.622826894124349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,48,2,128,0,1,float16,float16,0,7.207594553629558
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,48,4,128,0,1,float16,float16,0,6.993066787719727
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,3072,48,8,128,0,1,float16,float16,0,28.847274780273438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,48,1,128,0,1,float16,float16,0,3.5379199981689453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,48,48,128,0,1,float16,float16,0,15.212885538736979
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,48,8,128,0,1,float16,float16,0,7.193941116333008
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,48,1,128,0,1,float16,float16,0,1.8479785919189453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,48,2,128,0,1,float16,float16,0,3.570858637491862
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,48,2,128,0,1,float16,float16,0,1.850538730621338
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,48,4,128,0,1,float16,float16,0,3.5751253763834634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,48,4,128,0,1,float16,float16,0,1.850538730621338
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,48,8,128,0,1,float16,float16,0,3.5457706451416016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,48,48,128,0,1,float16,float16,0,3.7577387491861978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,48,48,128,0,1,float16,float16,0,1.915221373240153
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,48,1,128,0,1,float16,float16,0,1.0115413665771484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,48,2,128,0,1,float16,float16,0,1.01256529490153
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,48,8,128,0,1,float16,float16,0,1.851904074350993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,48,4,128,0,1,float16,float16,0,1.0101760228474934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,48,48,128,0,1,float16,float16,0,1.0245119730631511
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,48,8,128,0,1,float16,float16,0,1.0088106791178386
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,48,1,128,0,1,float16,float16,0,13.880661010742188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,48,2,128,0,1,float16,float16,0,14.122325897216797
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,48,4,128,0,1,float16,float16,0,13.702826182047525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,48,1,128,0,1,float16,float16,0,6.664021174112956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,2048,48,1,128,0,1,float16,float16,0,27.934720357259113
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,2048,48,2,128,0,1,float16,float16,0,27.974314371744793
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,2048,48,4,128,0,1,float16,float16,0,28.19464619954427
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,48,8,128,0,1,float16,float16,0,14.036479949951172
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,48,2,128,0,1,float16,float16,0,6.681770960489909
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,2048,48,8,128,0,1,float16,float16,0,28.261205037434895
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,48,48,128,0,1,float16,float16,0,7.462229410807292
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,48,4,128,0,1,float16,float16,0,6.902101516723633
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,48,1,128,0,1,float16,float16,0,3.3790292739868164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,48,2,128,0,1,float16,float16,0,3.3448959986368814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,48,8,128,0,1,float16,float16,0,6.826325098673503
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,48,48,128,0,1,float16,float16,0,14.719146728515625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,48,4,128,0,1,float16,float16,0,3.37937068939209
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,48,1,128,0,1,float16,float16,0,1.7274880409240723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,48,8,128,0,1,float16,float16,0,3.3821013768514
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,48,2,128,0,1,float16,float16,0,1.729365348815918
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,48,48,128,0,1,float16,float16,0,3.7099520365397134
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,48,4,128,0,1,float16,float16,0,1.7370452880859375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,48,1,128,0,1,float16,float16,0,0.9268906911214193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,48,2,128,0,1,float16,float16,0,0.9210879802703857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,48,48,128,0,1,float16,float16,0,1.8524160385131836
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,48,8,128,0,1,float16,float16,0,1.7394347190856934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,48,4,128,0,1,float16,float16,0,0.9236480394999186
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,48,1,128,0,1,float16,float16,0,0.520362655321757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,48,2,128,0,1,float16,float16,0,0.5220693349838257
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,48,8,128,0,1,float16,float16,0,0.9268906911214193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,48,48,128,0,1,float16,float16,0,0.9562453428904215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,48,48,128,0,1,float16,float16,0,0.5312853256861368
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,48,4,128,0,1,float16,float16,0,0.5218986670176188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,48,8,128,0,1,float16,float16,0,0.521727999051412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,48,1,128,0,1,float16,float16,0,8.336554845174154
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,48,2,128,0,1,float16,float16,0,8.401578903198242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,48,4,128,0,1,float16,float16,0,8.325802485148111
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,48,1,128,0,1,float16,float16,0,4.013397216796875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1536,48,1,128,0,1,float16,float16,0,16.606037139892578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1536,48,2,128,0,1,float16,float16,0,17.033045450846355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1536,48,4,128,0,1,float16,float16,0,16.997034708658855
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,48,2,128,0,1,float16,float16,0,4.007253328959147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,48,8,128,0,1,float16,float16,0,8.38587760925293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1536,48,8,128,0,1,float16,float16,0,17.171114603678387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,48,48,128,0,1,float16,float16,0,4.559189478556315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,48,4,128,0,1,float16,float16,0,4.028415997823079
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,48,1,128,0,1,float16,float16,0,2.03383461634318
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,48,8,128,0,1,float16,float16,0,4.107263882954915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,48,2,128,0,1,float16,float16,0,2.0468053817749023
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,48,4,128,0,1,float16,float16,0,2.0490239461263022
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,48,48,128,0,1,float16,float16,0,2.2505812644958496
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,48,1,128,0,1,float16,float16,0,1.0670080184936523
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,48,2,128,0,1,float16,float16,0,1.0653013388315837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,48,4,128,0,1,float16,float16,0,1.0653013388315837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,48,8,128,0,1,float16,float16,0,2.071551958719889
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,48,48,128,0,1,float16,float16,0,1.1448319753011067
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,48,1,128,0,1,float16,float16,0,0.5763413508733114
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,48,8,128,0,1,float16,float16,0,1.0728106498718262
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,48,2,128,0,1,float16,float16,0,0.5775359869003296
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,48,48,128,0,1,float16,float16,0,9.214805603027344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,48,4,128,0,1,float16,float16,0,0.5749760071436564
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,48,8,128,0,1,float16,float16,0,0.5782186587651571
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,48,48,128,0,1,float16,float16,0,0.6034773190816244
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,48,1,128,0,1,float16,float16,0,0.337066650390625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,48,2,128,0,1,float16,float16,0,0.33433600266774494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,48,48,128,0,1,float16,float16,0,0.33774932225545246
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,48,4,128,0,1,float16,float16,0,0.33382399876912433
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,48,8,128,0,1,float16,float16,0,0.3375786542892456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,48,1,128,0,1,float16,float16,0,7.931050618489583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,48,2,128,0,1,float16,float16,0,8.221696217854818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,48,4,128,0,1,float16,float16,0,8.339114507039389
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,48,1,128,0,1,float16,float16,0,4.024490674336751
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,1024,48,1,128,0,1,float16,float16,0,16.26794687906901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,1024,48,2,128,0,1,float16,float16,0,16.47854995727539
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,1024,48,4,128,0,1,float16,float16,0,16.86835225423177
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,48,8,128,0,1,float16,float16,0,8.264362970987955
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,48,2,128,0,1,float16,float16,0,4.051626523335774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,1024,48,8,128,0,1,float16,float16,0,16.698880513509113
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,48,48,128,0,1,float16,float16,0,4.7453867594401045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,48,4,128,0,1,float16,float16,0,4.093781471252441
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,48,1,128,0,1,float16,float16,0,2.019498666127523
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,48,8,128,0,1,float16,float16,0,4.154538790384929
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,48,2,128,0,1,float16,float16,0,2.024618625640869
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,48,4,128,0,1,float16,float16,0,2.0457812945048013
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,48,48,128,0,1,float16,float16,0,9.564160029093424
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,48,8,128,0,1,float16,float16,0,2.063360055287679
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,48,48,128,0,1,float16,float16,0,2.3801172574361167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,48,1,128,0,1,float16,float16,0,1.0216106573740642
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,48,2,128,0,1,float16,float16,0,1.0357759793599446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,48,4,128,0,1,float16,float16,0,1.0403839747111003
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,48,1,128,0,1,float16,float16,0,0.5321386655171713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,48,2,128,0,1,float16,float16,0,0.5376000006993612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,48,48,128,0,1,float16,float16,0,1.1818666458129883
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,48,8,128,0,1,float16,float16,0,1.0410666465759277
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,48,4,128,0,1,float16,float16,0,0.5398186842600504
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,48,8,128,0,1,float16,float16,0,0.5423786640167236
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,48,48,128,0,1,float16,float16,0,0.6087679862976074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,48,1,128,0,1,float16,float16,0,0.2850133379300435
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,48,2,128,0,1,float16,float16,0,0.285866657892863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,48,8,128,0,1,float16,float16,0,0.2913279930750529
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,48,48,128,0,1,float16,float16,0,0.17698132991790771
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,48,48,128,0,1,float16,float16,0,0.31914667288462323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,48,1,128,0,1,float16,float16,0,0.15923200050989786
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,48,2,128,0,1,float16,float16,0,0.1621333360671997
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,48,4,128,0,1,float16,float16,0,0.2892799973487854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,48,4,128,0,1,float16,float16,0,0.1628159979979197
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,48,8,128,0,1,float16,float16,0,0.16264533003171286
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,48,2,128,0,1,float16,float16,0,5.562197367350261
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,48,1,128,0,1,float16,float16,0,5.39784558614095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,48,4,128,0,1,float16,float16,0,5.6784210205078125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,48,1,128,0,1,float16,float16,0,2.701653480529785
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,512,48,1,128,0,1,float16,float16,0,10.927616119384766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,512,48,2,128,0,1,float16,float16,0,11.060223897298178
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,512,48,4,128,0,1,float16,float16,0,11.463850657145182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,48,2,128,0,1,float16,float16,0,2.738346735636393
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,48,8,128,0,1,float16,float16,0,5.7267201741536455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,512,48,8,128,0,1,float16,float16,0,11.610965728759766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,48,48,128,0,1,float16,float16,0,3.4471254348754883
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,48,1,128,0,1,float16,float16,0,1.363968054453532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,48,4,128,0,1,float16,float16,0,2.8117332458496094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,48,2,128,0,1,float16,float16,0,1.3815466562906902
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,48,8,128,0,1,float16,float16,0,2.844671885172526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,48,48,128,0,1,float16,float16,0,6.918485641479492
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,48,1,128,0,1,float16,float16,0,0.6959786415100098
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,48,4,128,0,1,float16,float16,0,1.4102187156677246
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,48,2,128,0,1,float16,float16,0,0.7014400164286295
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,48,4,128,0,1,float16,float16,0,0.7133866945902506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,48,8,128,0,1,float16,float16,0,1.422335942586263
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,48,48,128,0,1,float16,float16,0,0.8557226657867432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,48,1,128,0,1,float16,float16,0,0.36266668637593585
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,48,8,128,0,1,float16,float16,0,0.7174826463063558
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,48,48,128,0,1,float16,float16,0,1.7418239911397297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,48,48,128,0,1,float16,float16,0,0.4399786790211995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,48,2,128,0,1,float16,float16,0,0.36471466223398846
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,48,4,128,0,1,float16,float16,0,0.3705173333485921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,48,1,128,0,1,float16,float16,0,0.19541333119074503
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,48,2,128,0,1,float16,float16,0,0.1960960030555725
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,48,48,128,0,1,float16,float16,0,0.23125332593917847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,48,8,128,0,1,float16,float16,0,0.20036266247431436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,48,48,128,0,1,float16,float16,0,0.1276586651802063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,48,4,128,0,1,float16,float16,0,0.20684800545374551
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,48,1,128,0,1,float16,float16,0,0.10547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,48,2,128,0,1,float16,float16,0,0.10752000411351521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,48,4,128,0,1,float16,float16,0,0.10956799983978271
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,48,8,128,0,1,float16,float16,0,0.3717120091120402
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,48,8,128,0,1,float16,float16,0,0.11161599556605022
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,48,1,128,0,1,float16,float16,0,0.06775466601053874
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,48,48,128,0,1,float16,float16,0,0.07526400188604991
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,48,4,128,0,1,float16,float16,0,0.06843733290831248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,48,8,128,0,1,float16,float16,0,0.06946133573849995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,48,2,128,0,1,float16,float16,0,0.06741333504517873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,48,1,128,0,1,float16,float16,0,2.0548267364501953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,48,2,128,0,1,float16,float16,0,2.107562700907389
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,48,4,128,0,1,float16,float16,0,2.16866127649943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,48,1,128,0,1,float16,float16,0,1.043455998102824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,256,48,1,128,0,1,float16,float16,0,4.1260372797648115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,256,48,2,128,0,1,float16,float16,0,4.224170684814453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,256,48,4,128,0,1,float16,float16,0,4.35421880086263
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,48,8,128,0,1,float16,float16,0,2.1913599967956543
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,256,48,8,128,0,1,float16,float16,0,4.407466570536296
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,48,4,128,0,1,float16,float16,0,1.0801493326822917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,48,2,128,0,1,float16,float16,0,1.051477352778117
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,48,48,128,0,1,float16,float16,0,1.4225066502888997
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,48,48,128,0,1,float16,float16,0,2.824021339416504
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,48,1,128,0,1,float16,float16,0,0.5338453451792399
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,48,8,128,0,1,float16,float16,0,1.0980693499247234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,48,48,128,0,1,float16,float16,0,0.7326719760894775
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,48,2,128,0,1,float16,float16,0,0.5389653444290161
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,48,1,128,0,1,float16,float16,0,0.27801599105199176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,48,4,128,0,1,float16,float16,0,0.5587626695632935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,48,2,128,0,1,float16,float16,0,0.27989333868026733
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,48,8,128,0,1,float16,float16,0,0.5534720023473104
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,48,4,128,0,1,float16,float16,0,0.2850133379300435
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,48,1,128,0,1,float16,float16,0,0.1479680041472117
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,48,48,128,0,1,float16,float16,0,0.3819520076115926
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,48,2,128,0,1,float16,float16,0,0.15121066570281982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,48,8,128,0,1,float16,float16,0,0.2877440055211385
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,48,48,128,0,1,float16,float16,0,0.19831466674804688
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,48,4,128,0,1,float16,float16,0,0.1539413332939148
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,48,1,128,0,1,float16,float16,0,0.08140799899895985
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,48,8,128,0,1,float16,float16,0,0.17971199750900269
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,48,48,128,0,1,float16,float16,0,0.11161599556605022
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,48,2,128,0,1,float16,float16,0,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,48,4,128,0,1,float16,float16,0,0.08499200145403545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,48,8,128,0,1,float16,float16,0,0.08516266942024231
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,48,1,128,0,1,float16,float16,0,0.050517335534095764
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,48,2,128,0,1,float16,float16,0,0.05034666756788889
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,48,4,128,0,1,float16,float16,0,0.051370665431022644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,48,8,128,0,1,float16,float16,0,0.05085866649945577
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,48,48,128,0,1,float16,float16,0,0.056661332647005715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,48,48,128,0,1,float16,float16,0,0.03310933212439219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,48,1,128,0,1,float16,float16,0,0.032255999743938446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,48,2,128,0,1,float16,float16,0,0.032085334261258446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,48,4,128,0,1,float16,float16,0,0.03259733319282532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,48,8,128,0,1,float16,float16,0,0.032085334261258446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,48,1,128,0,1,float16,float16,0,0.8888320128122965
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,48,2,128,0,1,float16,float16,0,0.9023146629333496
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,48,4,128,0,1,float16,float16,0,0.9393493334452311
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,128,48,1,128,0,1,float16,float16,0,1.7675946553548176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,128,48,2,128,0,1,float16,float16,0,1.8191359837849934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,48,1,128,0,1,float16,float16,0,0.4510720173517863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,128,48,4,128,0,1,float16,float16,0,1.8691412607828777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,128,48,8,128,0,1,float16,float16,0,1.8986666997273762
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,48,8,128,0,1,float16,float16,0,0.9332053661346436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,48,48,128,0,1,float16,float16,0,1.338538646697998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,48,2,128,0,1,float16,float16,0,0.4626773198445638
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,48,4,128,0,1,float16,float16,0,0.47035733858744305
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,48,48,128,0,1,float16,float16,0,0.6828373273213705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,48,8,128,0,1,float16,float16,0,0.4746239980061849
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,48,1,128,0,1,float16,float16,0,0.237226665019989
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,48,48,128,0,1,float16,float16,0,0.3592533270517985
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,48,2,128,0,1,float16,float16,0,0.2404693365097046
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,48,1,128,0,1,float16,float16,0,0.1262933313846588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,48,4,128,0,1,float16,float16,0,0.2450773318608602
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,48,2,128,0,1,float16,float16,0,0.1293653349081675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,48,8,128,0,1,float16,float16,0,0.2476373314857483
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,48,4,128,0,1,float16,float16,0,0.13243732849756876
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,48,48,128,0,1,float16,float16,0,0.18773333231608072
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,48,8,128,0,1,float16,float16,0,0.13499733805656433
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,48,48,128,0,1,float16,float16,0,0.10035199920336406
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,48,1,128,0,1,float16,float16,0,0.06621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,48,2,128,0,1,float16,float16,0,0.09130666653315227
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,48,4,128,0,1,float16,float16,0,0.06980266670385997
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,48,8,128,0,1,float16,float16,0,0.07133866846561432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,48,1,128,0,1,float16,float16,0,0.03942399968703588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,48,48,128,0,1,float16,float16,0,0.04607999821503957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,48,2,128,0,1,float16,float16,0,0.039594667653242745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,48,4,128,0,1,float16,float16,0,0.040106666584809623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,48,8,128,0,1,float16,float16,0,0.040789333482583366
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,48,1,128,0,1,float16,float16,0,0.025429333249727886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,48,48,128,0,1,float16,float16,0,0.02679466704527537
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,48,2,128,0,1,float16,float16,0,0.025600001215934753
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,48,4,128,0,1,float16,float16,0,0.025770666698614757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,48,8,128,0,1,float16,float16,0,0.0264533335963885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,48,48,128,0,1,float16,float16,0,0.020309332758188248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,48,2,128,0,1,float16,float16,0,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,48,4,128,0,1,float16,float16,0,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,48,8,128,0,1,float16,float16,0,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,48,1,128,0,1,float16,float16,0,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,48,1,128,0,1,float16,float16,0,0.4519253174463908
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,48,2,128,0,1,float16,float16,0,0.47035733858744305
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,48,4,128,0,1,float16,float16,0,0.4853760004043579
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,64,48,2,128,0,1,float16,float16,0,0.9233067035675049
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,64,48,1,128,0,1,float16,float16,0,0.8690346876780192
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,64,48,4,128,0,1,float16,float16,0,0.9625600179036459
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,48,1,128,0,1,float16,float16,0,0.23552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,64,48,8,128,0,1,float16,float16,0,0.9821866353352865
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,48,48,128,0,1,float16,float16,0,0.6850559711456299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,48,8,128,0,1,float16,float16,0,0.49322664737701416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,48,2,128,0,1,float16,float16,0,0.24712532758712769
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,48,48,128,0,1,float16,float16,0,0.3541333278020223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,48,8,128,0,1,float16,float16,0,0.2606079975763957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,48,1,128,0,1,float16,float16,0,0.12970667084058127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,48,4,128,0,1,float16,float16,0,0.13806933164596558
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,48,2,128,0,1,float16,float16,0,0.13619200388590494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,48,4,128,0,1,float16,float16,0,0.254805326461792
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,48,48,128,0,1,float16,float16,0,0.18756266434987387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,48,8,128,0,1,float16,float16,0,0.13994666934013367
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,48,48,128,0,1,float16,float16,0,0.10547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,48,1,128,0,1,float16,float16,0,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,48,2,128,0,1,float16,float16,0,0.06673066814740498
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,48,1,128,0,1,float16,float16,0,0.03754666695992152
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,48,4,128,0,1,float16,float16,0,0.0682666649421056
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,48,8,128,0,1,float16,float16,0,0.07116800049940745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,48,2,128,0,1,float16,float16,0,0.03839999934037527
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,48,4,128,0,1,float16,float16,0,0.03942399968703588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,48,8,128,0,1,float16,float16,0,0.03942399968703588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,48,48,128,0,1,float16,float16,0,0.04642133414745331
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,48,48,128,0,1,float16,float16,0,0.02628266563018163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,48,1,128,0,1,float16,float16,0,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,48,2,128,0,1,float16,float16,0,0.025258667767047882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,48,4,128,0,1,float16,float16,0,0.025429333249727886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,48,8,128,0,1,float16,float16,0,0.025770666698614757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,48,48,128,0,1,float16,float16,0,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,48,1,128,0,1,float16,float16,0,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,48,4,128,0,1,float16,float16,0,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,48,1,128,0,1,float16,float16,0,0.017407999684413273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,48,8,128,0,1,float16,float16,0,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,48,48,128,0,1,float16,float16,0,0.017237332959969837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,48,2,128,0,1,float16,float16,0,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,48,2,128,0,1,float16,float16,0,0.017237332959969837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,48,4,128,0,1,float16,float16,0,0.017237332959969837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,48,8,128,0,1,float16,float16,0,0.017407999684413273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,48,1,128,0,1,float16,float16,0,0.25890133778254193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,32,48,1,128,0,1,float16,float16,0,0.5003946622212728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,48,2,128,0,1,float16,float16,0,0.26385066906611127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,32,48,2,128,0,1,float16,float16,0,0.5097813208897909
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,48,4,128,0,1,float16,float16,0,0.26897066831588745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,32,48,4,128,0,1,float16,float16,0,0.5186560153961182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,32,48,8,128,0,1,float16,float16,0,0.521727999051412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,48,48,128,0,1,float16,float16,0,0.3853653271993001
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,48,1,128,0,1,float16,float16,0,0.1372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,48,2,128,0,1,float16,float16,0,0.14165332913398743
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,48,8,128,0,1,float16,float16,0,0.2688000003496806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,48,8,128,0,1,float16,float16,0,0.14506666858990988
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,48,1,128,0,1,float16,float16,0,0.07167999943097432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,48,48,128,0,1,float16,float16,0,0.18824533621470133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,48,48,128,0,1,float16,float16,0,0.10188800096511841
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,48,4,128,0,1,float16,float16,0,0.14387200276056925
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,48,2,128,0,1,float16,float16,0,0.07321600119272868
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,48,8,128,0,1,float16,float16,0,0.0769706666469574
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,48,4,128,0,1,float16,float16,0,0.07492266595363617
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,48,1,128,0,1,float16,float16,0,0.04113066693147024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,48,2,128,0,1,float16,float16,0,0.04215466479460398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,48,48,128,0,1,float16,float16,0,0.04863999783992767
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,48,4,128,0,1,float16,float16,0,0.04215466479460398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,48,48,128,0,1,float16,float16,0,0.026965332527955372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,48,1,128,0,1,float16,float16,0,0.025941332181294758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,48,2,128,0,1,float16,float16,0,0.02679466704527537
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,48,8,128,0,1,float16,float16,0,0.043178667624791466
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,48,8,128,0,1,float16,float16,0,0.027306665976842243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,48,4,128,0,1,float16,float16,0,0.0266239990790685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,48,1,128,0,1,float16,float16,0,0.01791999985774358
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,48,48,128,0,1,float16,float16,0,0.018602666755517323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,48,2,128,0,1,float16,float16,0,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,48,4,128,0,1,float16,float16,0,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,48,8,128,0,1,float16,float16,0,0.018602666755517323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,48,1,128,0,1,float16,float16,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,48,2,128,0,1,float16,float16,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,48,48,128,0,1,float16,float16,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,48,4,128,0,1,float16,float16,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,48,8,128,0,1,float16,float16,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,48,48,128,0,1,float16,float16,0,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,48,1,128,0,1,float16,float16,0,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,48,2,128,0,1,float16,float16,0,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,48,4,128,0,1,float16,float16,0,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,48,8,128,0,1,float16,float16,0,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,16,48,2,128,0,1,float16,float16,0,0.3949226538340251
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,16,48,4,128,0,1,float16,float16,0,0.3991893529891968
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,16,48,1,128,0,1,float16,float16,0,0.39133866628011066
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,48,2,128,0,1,float16,float16,0,0.20497065782546997
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,16,48,8,128,0,1,float16,float16,0,0.40174933274586994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,48,4,128,0,1,float16,float16,0,0.20787199338277182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,48,1,128,0,1,float16,float16,0,0.20189867417017618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,48,48,128,0,1,float16,float16,0,0.23586134115854898
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,48,8,128,0,1,float16,float16,0,0.20821332931518555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,48,2,128,0,1,float16,float16,0,0.10427733262379964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,48,1,128,0,1,float16,float16,0,0.10274133086204529
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,48,4,128,0,1,float16,float16,0,0.10666666428248088
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,48,48,128,0,1,float16,float16,0,0.1237333317597707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,48,8,128,0,1,float16,float16,0,0.1083733340104421
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,48,48,128,0,1,float16,float16,0,0.06229333579540253
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,48,2,128,0,1,float16,float16,0,0.0580266664425532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,48,4,128,0,1,float16,float16,0,0.05853866537412008
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,48,1,128,0,1,float16,float16,0,0.034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,48,48,128,0,1,float16,float16,0,0.034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,48,4,128,0,1,float16,float16,0,0.03515733281771342
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,48,2,128,0,1,float16,float16,0,0.034815999368826546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,48,8,128,0,1,float16,float16,0,0.05922133227189382
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,48,1,128,0,1,float16,float16,0,0.05734399954477946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,48,8,128,0,1,float16,float16,0,0.03549866626660029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,48,1,128,0,1,float16,float16,0,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,48,2,128,0,1,float16,float16,0,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,48,48,128,0,1,float16,float16,0,0.022357332209746044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,48,8,128,0,1,float16,float16,0,0.022698665658632915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,48,4,128,0,1,float16,float16,0,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,48,48,128,0,1,float16,float16,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,48,2,128,0,1,float16,float16,0,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,48,1,128,0,1,float16,float16,0,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,48,4,128,0,1,float16,float16,0,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,48,8,128,0,1,float16,float16,0,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,48,48,128,0,1,float16,float16,0,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,48,1,128,0,1,float16,float16,0,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,48,2,128,0,1,float16,float16,0,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,48,4,128,0,1,float16,float16,0,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,48,8,128,0,1,float16,float16,0,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,48,1,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,48,2,128,0,1,float16,float16,0,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,48,4,128,0,1,float16,float16,0,0.013823999712864557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,48,8,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,48,48,128,0,1,float16,float16,0,0.013653332988421122
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,40,2,128,0,1,float16,float16,0,75.11756896972656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,40,1,128,0,1,float16,float16,0,74.40452067057292
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,40,4,128,0,1,float16,float16,0,74.20808410644531
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,40,1,128,0,1,float16,float16,0,36.78003184000651
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16384,40,2,128,0,1,float16,float16,0,147.56676228841147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16384,40,1,128,0,1,float16,float16,0,145.48821004231772
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16384,40,4,128,0,1,float16,float16,0,146.29444376627603
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,40,40,128,0,1,float16,float16,0,37.61681111653646
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,40,40,128,0,1,float16,float16,0,75.18498229980469
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,40,2,128,0,1,float16,float16,0,36.77252197265625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,40,8,128,0,1,float16,float16,0,74.82162984212239
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16384,40,8,128,0,1,float16,float16,0,148.56157430013022
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,40,1,128,0,1,float16,float16,0,18.913108825683594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,40,2,128,0,1,float16,float16,0,18.784085591634113
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,40,4,128,0,1,float16,float16,0,37.20755259195963
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,40,4,128,0,1,float16,float16,0,18.78271993001302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,40,8,128,0,1,float16,float16,0,36.371968587239586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,40,40,128,0,1,float16,float16,0,18.83016586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,40,8,128,0,1,float16,float16,0,19.010218302408855
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,40,1,128,0,1,float16,float16,0,41.13817596435547
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,40,2,128,0,1,float16,float16,0,41.64113108317057
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,40,4,128,0,1,float16,float16,0,42.23368581136068
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,12288,40,1,128,0,1,float16,float16,0,84.86912027994792
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,40,1,128,0,1,float16,float16,0,21.339818318684895
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,12288,40,2,128,0,1,float16,float16,0,85.03910319010417
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,12288,40,4,128,0,1,float16,float16,0,84.80990091959636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,40,2,128,0,1,float16,float16,0,21.25158437093099
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,40,40,128,0,1,float16,float16,0,42.53132883707682
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,40,40,128,0,1,float16,float16,0,21.152596791585285
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,12288,40,8,128,0,1,float16,float16,0,84.33476257324219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,40,4,128,0,1,float16,float16,0,21.296981811523438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,40,8,128,0,1,float16,float16,0,41.698646545410156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,40,1,128,0,1,float16,float16,0,10.914133707682291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,40,8,128,0,1,float16,float16,0,21.285717010498047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,40,2,128,0,1,float16,float16,0,10.81924311319987
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,40,40,128,0,1,float16,float16,0,11.006292978922525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,40,4,128,0,1,float16,float16,0,10.718037923177084
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,40,8,128,0,1,float16,float16,0,10.870783487955729
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,40,1,128,0,1,float16,float16,0,29.60315704345703
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,40,2,128,0,1,float16,float16,0,29.249537150065105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,40,4,128,0,1,float16,float16,0,29.60742441813151
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,10240,40,1,128,0,1,float16,float16,0,59.37715148925781
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,10240,40,2,128,0,1,float16,float16,0,58.2635498046875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,10240,40,4,128,0,1,float16,float16,0,59.633321126302086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,40,1,128,0,1,float16,float16,0,14.830251057942709
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,40,8,128,0,1,float16,float16,0,29.85181935628255
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,40,40,128,0,1,float16,float16,0,29.681493123372395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,40,2,128,0,1,float16,float16,0,15.003306070963541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,10240,40,8,128,0,1,float16,float16,0,59.60225931803385
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,40,4,128,0,1,float16,float16,0,14.60906728108724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,40,40,128,0,1,float16,float16,0,14.849194844563803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,40,1,128,0,1,float16,float16,0,7.671295801798503
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,40,2,128,0,1,float16,float16,0,7.599615732828776
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,40,8,128,0,1,float16,float16,0,15.0567258199056
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,40,40,128,0,1,float16,float16,0,7.699455897013347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,40,4,128,0,1,float16,float16,0,7.679829279581706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,40,8,128,0,1,float16,float16,0,7.669759750366211
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,40,1,128,0,1,float16,float16,0,38.22882080078125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,40,2,128,0,1,float16,float16,0,38.354601542154946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,40,4,128,0,1,float16,float16,0,38.664703369140625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,40,1,128,0,1,float16,float16,0,19.267925262451172
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,8192,40,2,128,0,1,float16,float16,0,76.13132731119792
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,8192,40,1,128,0,1,float16,float16,0,75.79989115397136
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,8192,40,4,128,0,1,float16,float16,0,77.41747029622395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,40,2,128,0,1,float16,float16,0,19.258538564046223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,40,8,128,0,1,float16,float16,0,38.37576548258463
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,40,40,128,0,1,float16,float16,0,39.39788818359375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,40,40,128,0,1,float16,float16,0,19.70483144124349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,40,4,128,0,1,float16,float16,0,18.90560022989909
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,8192,40,8,128,0,1,float16,float16,0,76.45081583658855
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,40,1,128,0,1,float16,float16,0,9.686869303385416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,40,2,128,0,1,float16,float16,0,9.757183710734049
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,40,1,128,0,1,float16,float16,0,4.946943918863933
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,40,8,128,0,1,float16,float16,0,18.935296376546223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,40,4,128,0,1,float16,float16,0,9.789951960245768
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,40,8,128,0,1,float16,float16,0,9.829546610514322
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,40,2,128,0,1,float16,float16,0,5.03057066599528
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,40,40,128,0,1,float16,float16,0,4.973397254943848
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,40,4,128,0,1,float16,float16,0,5.030229250590007
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,40,8,128,0,1,float16,float16,0,4.942847887674968
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,40,40,128,0,1,float16,float16,0,9.914368311564127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,40,1,128,0,1,float16,float16,0,22.008148193359375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,40,2,128,0,1,float16,float16,0,21.889195760091145
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,40,4,128,0,1,float16,float16,0,22.179669698079426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,6144,40,1,128,0,1,float16,float16,0,43.485015869140625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,6144,40,2,128,0,1,float16,float16,0,44.310699462890625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,40,1,128,0,1,float16,float16,0,11.077290852864584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,6144,40,4,128,0,1,float16,float16,0,44.5644785563151
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,40,2,128,0,1,float16,float16,0,10.792106628417969
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,6144,40,8,128,0,1,float16,float16,0,44.51362101236979
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,40,4,128,0,1,float16,float16,0,11.080020904541016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,40,40,128,0,1,float16,float16,0,11.275946299235025
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,40,8,128,0,1,float16,float16,0,22.0586675008138
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,40,40,128,0,1,float16,float16,0,22.410921732584637
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,40,1,128,0,1,float16,float16,0,5.472597122192383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,40,2,128,0,1,float16,float16,0,5.63968022664388
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,40,1,128,0,1,float16,float16,0,2.8881918589274087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,40,4,128,0,1,float16,float16,0,5.636778513590495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,40,8,128,0,1,float16,float16,0,5.550079981486003
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,40,8,128,0,1,float16,float16,0,11.153237660725912
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,40,40,128,0,1,float16,float16,0,5.769898732503255
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,40,40,128,0,1,float16,float16,0,2.920618693033854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,40,2,128,0,1,float16,float16,0,2.8702720006306968
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,40,4,128,0,1,float16,float16,0,2.890752156575521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,40,8,128,0,1,float16,float16,0,2.8842665354410806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,40,1,128,0,1,float16,float16,0,19.950592041015625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,40,2,128,0,1,float16,float16,0,20.54092788696289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,40,4,128,0,1,float16,float16,0,20.227413177490234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,40,1,128,0,1,float16,float16,0,10.043733596801758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,4096,40,1,128,0,1,float16,float16,0,41.36226145426432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,4096,40,2,128,0,1,float16,float16,0,40.733184814453125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,4096,40,4,128,0,1,float16,float16,0,41.42540740966797
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,40,2,128,0,1,float16,float16,0,10.063018798828125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,40,4,128,0,1,float16,float16,0,10.182997385660807
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,40,8,128,0,1,float16,float16,0,20.180309295654297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,40,40,128,0,1,float16,float16,0,10.74176025390625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,4096,40,8,128,0,1,float16,float16,0,41.40373229980469
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,40,40,128,0,1,float16,float16,0,21.300735473632812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,40,1,128,0,1,float16,float16,0,5.058389345804851
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,40,2,128,0,1,float16,float16,0,5.127338727315267
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,40,8,128,0,1,float16,float16,0,10.2205441792806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,40,2,128,0,1,float16,float16,0,2.5951573053995767
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,40,1,128,0,1,float16,float16,0,2.6077866554260254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,40,4,128,0,1,float16,float16,0,5.034495989481608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,40,8,128,0,1,float16,float16,0,5.153962771097819
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,40,40,128,0,1,float16,float16,0,5.337429046630859
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,40,40,128,0,1,float16,float16,0,2.6774187088012695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,40,2,128,0,1,float16,float16,0,1.3982720375061035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,40,4,128,0,1,float16,float16,0,2.598741372426351
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,40,1,128,0,1,float16,float16,0,1.4124372800191243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,40,4,128,0,1,float16,float16,0,1.4119253158569336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,40,8,128,0,1,float16,float16,0,2.601130644480387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,40,8,128,0,1,float16,float16,0,1.4105599721272786
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,40,40,128,0,1,float16,float16,0,1.4158506393432617
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,40,1,128,0,1,float16,float16,0,11.848021189371744
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,40,2,128,0,1,float16,float16,0,12.159659067789713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,40,4,128,0,1,float16,float16,0,11.898709615071615
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,40,1,128,0,1,float16,float16,0,5.85096549987793
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,3072,40,1,128,0,1,float16,float16,0,24.10888417561849
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,3072,40,2,128,0,1,float16,float16,0,23.812950134277344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,3072,40,4,128,0,1,float16,float16,0,24.354644775390625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,40,2,128,0,1,float16,float16,0,5.838677088419597
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,40,8,128,0,1,float16,float16,0,12.059989929199219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,3072,40,8,128,0,1,float16,float16,0,24.403114318847656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,40,4,128,0,1,float16,float16,0,6.016511917114258
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,40,40,128,0,1,float16,float16,0,12.666197458902994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,40,1,128,0,1,float16,float16,0,2.954239845275879
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,40,40,128,0,1,float16,float16,0,6.339754740397136
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,40,2,128,0,1,float16,float16,0,2.9579947789510093
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,40,8,128,0,1,float16,float16,0,5.86137580871582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,40,1,128,0,1,float16,float16,0,1.557162602742513
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,40,4,128,0,1,float16,float16,0,2.997589429219564
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,40,2,128,0,1,float16,float16,0,1.5561386744181316
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,40,40,128,0,1,float16,float16,0,3.1505066553751626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,40,8,128,0,1,float16,float16,0,3.00492795308431
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,40,4,128,0,1,float16,float16,0,1.559893290201823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,40,8,128,0,1,float16,float16,0,1.561087926228841
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,40,40,128,0,1,float16,float16,0,1.6047786076863606
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,40,1,128,0,1,float16,float16,0,0.8553813298543295
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,40,2,128,0,1,float16,float16,0,0.8623786767323812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,40,4,128,0,1,float16,float16,0,0.8581120173136393
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,40,8,128,0,1,float16,float16,0,0.8606719970703125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,40,40,128,0,1,float16,float16,0,0.8700586954752604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,40,1,128,0,1,float16,float16,0,11.679232279459635
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,40,2,128,0,1,float16,float16,0,11.757909138997396
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,40,4,128,0,1,float16,float16,0,11.532459259033203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,40,1,128,0,1,float16,float16,0,5.580970764160156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,2048,40,1,128,0,1,float16,float16,0,23.13318379720052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,2048,40,2,128,0,1,float16,float16,0,23.058430989583332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,2048,40,4,128,0,1,float16,float16,0,23.552169799804688
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,40,8,128,0,1,float16,float16,0,11.782997131347656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,40,2,128,0,1,float16,float16,0,5.603498458862305
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,40,4,128,0,1,float16,float16,0,5.656746546427409
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,40,40,128,0,1,float16,float16,0,6.257834752400716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,40,1,128,0,1,float16,float16,0,2.8076372146606445
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,2048,40,8,128,0,1,float16,float16,0,23.438336690266926
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,40,2,128,0,1,float16,float16,0,2.7973972956339517
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,40,40,128,0,1,float16,float16,0,12.51925277709961
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,40,8,128,0,1,float16,float16,0,5.649920145670573
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,40,4,128,0,1,float16,float16,0,2.8076372146606445
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,40,1,128,0,1,float16,float16,0,1.458346684773763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,40,2,128,0,1,float16,float16,0,1.451690673828125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,40,4,128,0,1,float16,float16,0,1.4612479209899902
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,40,40,128,0,1,float16,float16,0,3.0472532908121743
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,40,8,128,0,1,float16,float16,0,2.8530346552530923
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,40,2,128,0,1,float16,float16,0,0.7753386497497559
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,40,1,128,0,1,float16,float16,0,0.784554640452067
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,40,8,128,0,1,float16,float16,0,1.4626132647196453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,40,40,128,0,1,float16,float16,0,1.5426559448242188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,40,4,128,0,1,float16,float16,0,0.7825067043304443
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,40,8,128,0,1,float16,float16,0,0.7852373123168945
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,40,40,128,0,1,float16,float16,0,0.8075946966807047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,40,40,128,0,1,float16,float16,0,0.4527786572774251
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,40,1,128,0,1,float16,float16,0,0.4416853189468384
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,40,2,128,0,1,float16,float16,0,0.4457813501358032
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,40,4,128,0,1,float16,float16,0,0.4479999939600627
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,40,8,128,0,1,float16,float16,0,0.44492801030476886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,40,1,128,0,1,float16,float16,0,6.969514846801758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,40,2,128,0,1,float16,float16,0,6.9056854248046875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,40,4,128,0,1,float16,float16,0,7.029589335123698
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,40,1,128,0,1,float16,float16,0,3.372031847635905
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1536,40,1,128,0,1,float16,float16,0,14.08853276570638
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1536,40,2,128,0,1,float16,float16,0,14.277460734049479
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1536,40,4,128,0,1,float16,float16,0,14.040064493815104
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,40,2,128,0,1,float16,float16,0,3.3902934392293296
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,40,8,128,0,1,float16,float16,0,7.043071746826172
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1536,40,8,128,0,1,float16,float16,0,14.422528584798178
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,40,4,128,0,1,float16,float16,0,3.422549247741699
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,40,40,128,0,1,float16,float16,0,7.49567985534668
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,40,1,128,0,1,float16,float16,0,1.7032532691955566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,40,40,128,0,1,float16,float16,0,3.7592748006184897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,40,8,128,0,1,float16,float16,0,3.4194774627685547
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,40,2,128,0,1,float16,float16,0,1.7215147018432617
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,40,1,128,0,1,float16,float16,0,0.8953173160552979
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,40,4,128,0,1,float16,float16,0,1.7203200658162434
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,40,40,128,0,1,float16,float16,0,1.8826239903767903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,40,2,128,0,1,float16,float16,0,0.897536039352417
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,40,4,128,0,1,float16,float16,0,0.9002666473388672
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,40,8,128,0,1,float16,float16,0,1.7254400253295898
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,40,40,128,0,1,float16,float16,0,0.9687039852142334
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,40,8,128,0,1,float16,float16,0,0.9057280222574869
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,40,2,128,0,1,float16,float16,0,0.4916906754175822
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,40,1,128,0,1,float16,float16,0,0.4848639965057373
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,40,4,128,0,1,float16,float16,0,0.490666667620341
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,40,1,128,0,1,float16,float16,0,0.2841599980990092
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,40,40,128,0,1,float16,float16,0,0.2916693290074666
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,40,8,128,0,1,float16,float16,0,0.490666667620341
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,40,40,128,0,1,float16,float16,0,0.5094399849573771
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,40,2,128,0,1,float16,float16,0,0.2879146734873454
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,40,4,128,0,1,float16,float16,0,0.28484266996383667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,40,8,128,0,1,float16,float16,0,0.2872320016225179
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,40,1,128,0,1,float16,float16,0,6.760618845621745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,40,2,128,0,1,float16,float16,0,6.783317565917969
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,40,4,128,0,1,float16,float16,0,6.936234792073567
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,40,1,128,0,1,float16,float16,0,3.357184092203776
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,1024,40,2,128,0,1,float16,float16,0,13.64462916056315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,1024,40,1,128,0,1,float16,float16,0,13.60742441813151
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,1024,40,4,128,0,1,float16,float16,0,13.962752024332682
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,40,2,128,0,1,float16,float16,0,3.3653761545817056
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,40,8,128,0,1,float16,float16,0,7.027711868286133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,1024,40,8,128,0,1,float16,float16,0,14.290261586507162
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,40,40,128,0,1,float16,float16,0,8.030037562052408
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,40,4,128,0,1,float16,float16,0,3.37664000193278
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,40,1,128,0,1,float16,float16,0,1.7029120127360027
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,40,2,128,0,1,float16,float16,0,1.70632537206014
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,40,40,128,0,1,float16,float16,0,3.940693219502767
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,40,8,128,0,1,float16,float16,0,3.4391040802001953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,40,40,128,0,1,float16,float16,0,1.9858773549397786
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,40,4,128,0,1,float16,float16,0,1.7047893206278484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,40,8,128,0,1,float16,float16,0,1.7300480206807454
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,40,1,128,0,1,float16,float16,0,0.8611839612325033
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,40,2,128,0,1,float16,float16,0,0.8673280080159506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,40,4,128,0,1,float16,float16,0,0.8705706596374512
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,40,1,128,0,1,float16,float16,0,0.4493653376897176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,40,2,128,0,1,float16,float16,0,0.4505600134531657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,40,8,128,0,1,float16,float16,0,0.8866133689880371
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,40,40,128,0,1,float16,float16,0,1.0026666323343914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,40,4,128,0,1,float16,float16,0,0.45499734083811444
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,40,40,128,0,1,float16,float16,0,0.544597347577413
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,40,8,128,0,1,float16,float16,0,0.46062934398651123
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,40,40,128,0,1,float16,float16,0,0.2955946723620097
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,40,1,128,0,1,float16,float16,0,0.24337067206700644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,40,2,128,0,1,float16,float16,0,0.24541866779327393
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,40,4,128,0,1,float16,float16,0,0.24541866779327393
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,40,1,128,0,1,float16,float16,0,0.13806933164596558
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,40,40,128,0,1,float16,float16,0,0.15598932902018228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,40,4,128,0,1,float16,float16,0,0.13943466544151306
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,40,2,128,0,1,float16,float16,0,0.14011733730634054
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,40,8,128,0,1,float16,float16,0,0.24900267521540323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,40,8,128,0,1,float16,float16,0,0.14028799533843994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,40,1,128,0,1,float16,float16,0,4.588202794392903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,40,2,128,0,1,float16,float16,0,4.592639923095703
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,40,4,128,0,1,float16,float16,0,4.66158930460612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,512,40,1,128,0,1,float16,float16,0,9.36465072631836
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,40,1,128,0,1,float16,float16,0,2.276693344116211
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,512,40,2,128,0,1,float16,float16,0,9.264810562133789
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,512,40,4,128,0,1,float16,float16,0,9.526442845662435
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,40,8,128,0,1,float16,float16,0,4.78600533803304
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,40,2,128,0,1,float16,float16,0,2.3016106287638345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,512,40,8,128,0,1,float16,float16,0,9.795413335164389
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,40,4,128,0,1,float16,float16,0,2.306389331817627
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,40,40,128,0,1,float16,float16,0,2.8904107411702475
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,40,2,128,0,1,float16,float16,0,1.1596799691518147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,40,1,128,0,1,float16,float16,0,1.155413309733073
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,40,40,128,0,1,float16,float16,0,5.785600026448567
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,40,8,128,0,1,float16,float16,0,2.3700480461120605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,40,4,128,0,1,float16,float16,0,1.1675306955973308
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,40,8,128,0,1,float16,float16,0,1.190399964650472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,40,1,128,0,1,float16,float16,0,0.5847040017445883
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,40,2,128,0,1,float16,float16,0,0.590336004892985
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,40,4,128,0,1,float16,float16,0,0.5957973400751749
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,40,40,128,0,1,float16,float16,0,1.4487892786661785
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,40,8,128,0,1,float16,float16,0,0.6089386542638143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,40,2,128,0,1,float16,float16,0,0.30873600641886395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,40,1,128,0,1,float16,float16,0,0.30822400252024335
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,40,40,128,0,1,float16,float16,0,0.7299413681030273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,40,4,128,0,1,float16,float16,0,0.31112533807754517
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,40,8,128,0,1,float16,float16,0,0.3150506615638733
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,40,1,128,0,1,float16,float16,0,0.16588800152142844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,40,40,128,0,1,float16,float16,0,0.374783992767334
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,40,2,128,0,1,float16,float16,0,0.16571733355522156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,40,4,128,0,1,float16,float16,0,0.16810667514801025
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,40,8,128,0,1,float16,float16,0,0.169813334941864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,40,1,128,0,1,float16,float16,0,0.09335466225941975
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,40,40,128,0,1,float16,float16,0,0.11059199770291646
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,40,2,128,0,1,float16,float16,0,0.09489066402117412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,40,40,128,0,1,float16,float16,0,0.20189867417017618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,40,4,128,0,1,float16,float16,0,0.09454933802286784
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,40,8,128,0,1,float16,float16,0,0.0942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,40,40,128,0,1,float16,float16,0,0.0628053347269694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,40,1,128,0,1,float16,float16,0,0.058880001306533813
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,40,2,128,0,1,float16,float16,0,0.05870933334032694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,40,4,128,0,1,float16,float16,0,0.05819733440876007
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,40,8,128,0,1,float16,float16,0,0.05973333120346069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,40,1,128,0,1,float16,float16,0,1.7455786069234211
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,40,2,128,0,1,float16,float16,0,1.7640105883280437
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,40,4,128,0,1,float16,float16,0,1.7822720209757488
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,256,40,1,128,0,1,float16,float16,0,3.51146666208903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,256,40,2,128,0,1,float16,float16,0,3.5676161448160806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,40,1,128,0,1,float16,float16,0,0.8816640377044678
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,256,40,4,128,0,1,float16,float16,0,3.5792214075724282
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,40,8,128,0,1,float16,float16,0,1.8213547070821126
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,40,2,128,0,1,float16,float16,0,0.8912213643391927
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,256,40,8,128,0,1,float16,float16,0,3.710293451944987
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,40,40,128,0,1,float16,float16,0,2.37499729792277
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,40,40,128,0,1,float16,float16,0,1.1980799833933513
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,40,4,128,0,1,float16,float16,0,0.8987306753794352
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,40,8,128,0,1,float16,float16,0,0.9175039927164713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,40,1,128,0,1,float16,float16,0,0.4503893454869588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,40,1,128,0,1,float16,float16,0,0.23552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,40,2,128,0,1,float16,float16,0,0.4505600134531657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,40,40,128,0,1,float16,float16,0,0.6208853324254354
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,40,4,128,0,1,float16,float16,0,0.4585813283920288
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,40,2,128,0,1,float16,float16,0,0.23808000485102335
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,40,8,128,0,1,float16,float16,0,0.46984533468882245
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,40,40,128,0,1,float16,float16,0,0.32290132840474445
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,40,4,128,0,1,float16,float16,0,0.23910399278004965
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,40,1,128,0,1,float16,float16,0,0.12680533528327942
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,40,4,128,0,1,float16,float16,0,0.12851199507713318
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,40,2,128,0,1,float16,float16,0,0.12714667121569315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,40,1,128,0,1,float16,float16,0,0.06997333467006683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,40,40,128,0,1,float16,float16,0,0.16691199938456217
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,40,8,128,0,1,float16,float16,0,0.2435413400332133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,40,40,128,0,1,float16,float16,0,0.08891733487447102
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,40,8,128,0,1,float16,float16,0,0.13107200463612875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,40,2,128,0,1,float16,float16,0,0.06997333467006683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,40,4,128,0,1,float16,float16,0,0.07014399766921997
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,40,8,128,0,1,float16,float16,0,0.07202133536338806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,40,1,128,0,1,float16,float16,0,0.043178667624791466
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,40,4,128,0,1,float16,float16,0,0.04249600072701772
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,40,40,128,0,1,float16,float16,0,0.046762665112813316
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,40,2,128,0,1,float16,float16,0,0.043178667624791466
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,40,8,128,0,1,float16,float16,0,0.042837331692377724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,40,40,128,0,1,float16,float16,0,0.03242666771014532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,40,1,128,0,1,float16,float16,0,0.031061333914597828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,40,2,128,0,1,float16,float16,0,0.030720000465710957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,40,8,128,0,1,float16,float16,0,0.03089066594839096
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,40,4,128,0,1,float16,float16,0,0.030207999050617218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,40,1,128,0,1,float16,float16,0,0.7589546839396158
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,40,2,128,0,1,float16,float16,0,0.7679999669392904
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,40,4,128,0,1,float16,float16,0,0.7751680215199789
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,128,40,1,128,0,1,float16,float16,0,1.5196159680684407
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,128,40,2,128,0,1,float16,float16,0,1.5363413492838542
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,128,40,4,128,0,1,float16,float16,0,1.5528960227966309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,40,1,128,0,1,float16,float16,0,0.3838293155034383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,128,40,8,128,0,1,float16,float16,0,1.609557310740153
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,40,8,128,0,1,float16,float16,0,0.7999146779378256
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,40,2,128,0,1,float16,float16,0,0.38997332255045575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,40,40,128,0,1,float16,float16,0,1.1415893236796062
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,40,4,128,0,1,float16,float16,0,0.39611732959747314
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,40,40,128,0,1,float16,float16,0,0.5876053174336752
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,40,1,128,0,1,float16,float16,0,0.20053333044052124
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,40,8,128,0,1,float16,float16,0,0.41198933124542236
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,40,40,128,0,1,float16,float16,0,0.2993493278821309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,40,2,128,0,1,float16,float16,0,0.20258132616678873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,40,4,128,0,1,float16,float16,0,0.2039466698964437
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,40,8,128,0,1,float16,float16,0,0.21230934063593546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,40,1,128,0,1,float16,float16,0,0.10615467031796773
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,40,40,128,0,1,float16,float16,0,0.15752533078193665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,40,2,128,0,1,float16,float16,0,0.105813334385554
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,40,4,128,0,1,float16,float16,0,0.10752000411351521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,40,1,128,0,1,float16,float16,0,0.05597866574923197
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,40,8,128,0,1,float16,float16,0,0.11212799946467082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,40,40,128,0,1,float16,float16,0,0.07935999830563863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,40,2,128,0,1,float16,float16,0,0.05597866574923197
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,40,4,128,0,1,float16,float16,0,0.05614933371543884
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,40,8,128,0,1,float16,float16,0,0.0580266664425532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,40,40,128,0,1,float16,float16,0,0.038912000755469
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,40,1,128,0,1,float16,float16,0,0.03549866626660029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,40,8,128,0,1,float16,float16,0,0.03566933423280716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,40,4,128,0,1,float16,float16,0,0.03515733281771342
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,40,2,128,0,1,float16,float16,0,0.03618133316437403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,40,40,128,0,1,float16,float16,0,0.0264533335963885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,40,1,128,0,1,float16,float16,0,0.025429333249727886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,40,2,128,0,1,float16,float16,0,0.024746666351954143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,40,4,128,0,1,float16,float16,0,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,40,1,128,0,1,float16,float16,0,0.01877333347996076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,40,40,128,0,1,float16,float16,0,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,40,2,128,0,1,float16,float16,0,0.01877333347996076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,40,8,128,0,1,float16,float16,0,0.02491733431816101
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,40,4,128,0,1,float16,float16,0,0.01877333347996076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,40,8,128,0,1,float16,float16,0,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,40,1,128,0,1,float16,float16,0,0.37717334429423016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,40,2,128,0,1,float16,float16,0,0.3887786865234375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,40,4,128,0,1,float16,float16,0,0.3985066811243693
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,64,40,1,128,0,1,float16,float16,0,0.7285760243733724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,64,40,2,128,0,1,float16,float16,0,0.7592960198720297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,64,40,4,128,0,1,float16,float16,0,0.7801173528035482
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,64,40,8,128,0,1,float16,float16,0,0.814250628153483
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,40,1,128,0,1,float16,float16,0,0.19848533471425375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,40,2,128,0,1,float16,float16,0,0.202239990234375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,40,40,128,0,1,float16,float16,0,0.585045337677002
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,40,4,128,0,1,float16,float16,0,0.20906666914621988
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,40,8,128,0,1,float16,float16,0,0.41437868277231854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,40,8,128,0,1,float16,float16,0,0.2167466680208842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,40,40,128,0,1,float16,float16,0,0.3002026677131653
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,40,1,128,0,1,float16,float16,0,0.10461866855621338
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,40,2,128,0,1,float16,float16,0,0.105813334385554
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,40,4,128,0,1,float16,float16,0,0.10905599594116211
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,40,40,128,0,1,float16,float16,0,0.16127999623616537
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,40,8,128,0,1,float16,float16,0,0.11281067132949829
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,40,40,128,0,1,float16,float16,0,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,40,1,128,0,1,float16,float16,0,0.05495466788609823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,40,2,128,0,1,float16,float16,0,0.05529599885145823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,40,4,128,0,1,float16,float16,0,0.0554666668176651
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,40,8,128,0,1,float16,float16,0,0.05751466751098633
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,40,1,128,0,1,float16,float16,0,0.03549866626660029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,40,2,128,0,1,float16,float16,0,0.03532800078392029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,40,40,128,0,1,float16,float16,0,0.040448000033696495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,40,4,128,0,1,float16,float16,0,0.03515733281771342
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,40,40,128,0,1,float16,float16,0,0.025258667767047882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,40,1,128,0,1,float16,float16,0,0.023893333971500397
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,40,8,128,0,1,float16,float16,0,0.03532800078392029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,40,2,128,0,1,float16,float16,0,0.023893333971500397
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,40,4,128,0,1,float16,float16,0,0.023893333971500397
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,40,8,128,0,1,float16,float16,0,0.024234667420387268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,40,40,128,0,1,float16,float16,0,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,40,1,128,0,1,float16,float16,0,0.01877333347996076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,40,2,128,0,1,float16,float16,0,0.018602666755517323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,40,4,128,0,1,float16,float16,0,0.018432000031073887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,40,40,128,0,1,float16,float16,0,0.0170666662355264
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,40,8,128,0,1,float16,float16,0,0.01877333347996076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,40,1,128,0,1,float16,float16,0,0.017407999684413273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,40,2,128,0,1,float16,float16,0,0.017237332959969837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,40,4,128,0,1,float16,float16,0,0.0170666662355264
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,40,8,128,0,1,float16,float16,0,0.017407999684413273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,40,1,128,0,1,float16,float16,0,0.22050132354100546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,40,2,128,0,1,float16,float16,0,0.22340265909830728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,32,40,1,128,0,1,float16,float16,0,0.4275199969609578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,32,40,2,128,0,1,float16,float16,0,0.43383467197418213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,40,4,128,0,1,float16,float16,0,0.22528000672658285
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,32,40,4,128,0,1,float16,float16,0,0.43622398376464844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,40,40,128,0,1,float16,float16,0,0.3063466747601827
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,32,40,8,128,0,1,float16,float16,0,0.4493653376897176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,40,1,128,0,1,float16,float16,0,0.11485866705576579
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,40,2,128,0,1,float16,float16,0,0.11622400085131328
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,40,8,128,0,1,float16,float16,0,0.23244800170262656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,40,40,128,0,1,float16,float16,0,0.15974400440851846
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,40,8,128,0,1,float16,float16,0,0.12134400010108948
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,40,4,128,0,1,float16,float16,0,0.11758933464686076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,40,2,128,0,1,float16,float16,0,0.06007466713587443
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,40,40,128,0,1,float16,float16,0,0.08226133386294048
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,40,1,128,0,1,float16,float16,0,0.059562668204307556
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,40,4,128,0,1,float16,float16,0,0.059903999169667564
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,40,1,128,0,1,float16,float16,0,0.034815999368826546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,40,8,128,0,1,float16,float16,0,0.0631466656923294
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,40,40,128,0,1,float16,float16,0,0.038912000755469
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,40,2,128,0,1,float16,float16,0,0.03515733281771342
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,40,8,128,0,1,float16,float16,0,0.03515733281771342
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,40,4,128,0,1,float16,float16,0,0.03498666733503342
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,40,1,128,0,1,float16,float16,0,0.025600001215934753
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,40,2,128,0,1,float16,float16,0,0.025600001215934753
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,40,40,128,0,1,float16,float16,0,0.02611200014750163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,40,8,128,0,1,float16,float16,0,0.025941332181294758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,40,4,128,0,1,float16,float16,0,0.025429333249727886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,40,40,128,0,1,float16,float16,0,0.01757866640885671
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,40,1,128,0,1,float16,float16,0,0.017407999684413273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,40,2,128,0,1,float16,float16,0,0.017407999684413273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,40,4,128,0,1,float16,float16,0,0.017407999684413273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,40,8,128,0,1,float16,float16,0,0.01757866640885671
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,40,1,128,0,1,float16,float16,0,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,40,2,128,0,1,float16,float16,0,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,40,40,128,0,1,float16,float16,0,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,40,4,128,0,1,float16,float16,0,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,40,8,128,0,1,float16,float16,0,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,40,1,128,0,1,float16,float16,0,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,40,2,128,0,1,float16,float16,0,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,40,4,128,0,1,float16,float16,0,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,40,8,128,0,1,float16,float16,0,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,40,40,128,0,1,float16,float16,0,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,40,1,128,0,1,float16,float16,0,0.16844799121220908
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,16,40,1,128,0,1,float16,float16,0,0.3324586749076843
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,40,2,128,0,1,float16,float16,0,0.1704960068066915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,16,40,2,128,0,1,float16,float16,0,0.33553067843119305
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,16,40,4,128,0,1,float16,float16,0,0.36522666613260907
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,16,40,8,128,0,1,float16,float16,0,0.3449173370997111
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,40,40,128,0,1,float16,float16,0,0.1991680065790812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,40,1,128,0,1,float16,float16,0,0.08533333738644917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,40,8,128,0,1,float16,float16,0,0.17629865805308023
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,40,40,128,0,1,float16,float16,0,0.10274133086204529
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,40,2,128,0,1,float16,float16,0,0.08550399541854858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,40,8,128,0,1,float16,float16,0,0.08942932883898418
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,40,40,128,0,1,float16,float16,0,0.051029334465662636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,40,4,128,0,1,float16,float16,0,0.17254400253295898
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,40,4,128,0,1,float16,float16,0,0.08669867118199666
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,40,1,128,0,1,float16,float16,0,0.04761599997679392
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,40,2,128,0,1,float16,float16,0,0.04761599997679392
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,40,4,128,0,1,float16,float16,0,0.04778666794300079
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,40,8,128,0,1,float16,float16,0,0.04795733094215393
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,40,1,128,0,1,float16,float16,0,0.030037333567937214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,40,40,128,0,1,float16,float16,0,0.03054933249950409
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,40,1,128,0,1,float16,float16,0,0.021674667795499165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,40,8,128,0,1,float16,float16,0,0.030720000465710957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,40,40,128,0,1,float16,float16,0,0.021333334346612293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,40,4,128,0,1,float16,float16,0,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,40,2,128,0,1,float16,float16,0,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,40,4,128,0,1,float16,float16,0,0.030037333567937214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,40,8,128,0,1,float16,float16,0,0.022015998760859173
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,40,40,128,0,1,float16,float16,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,40,2,128,0,1,float16,float16,0,0.030037333567937214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,40,1,128,0,1,float16,float16,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,40,2,128,0,1,float16,float16,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,40,4,128,0,1,float16,float16,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,40,8,128,0,1,float16,float16,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,40,1,128,0,1,float16,float16,0,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,40,2,128,0,1,float16,float16,0,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,40,8,128,0,1,float16,float16,0,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,40,40,128,0,1,float16,float16,0,0.013653332988421122
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,40,40,128,0,1,float16,float16,0,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,40,1,128,0,1,float16,float16,0,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,40,2,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,40,4,128,0,1,float16,float16,0,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,40,8,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,40,4,128,0,1,float16,float16,0,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,32,1,128,0,1,float16,float16,0,59.056640625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,32,2,128,0,1,float16,float16,0,57.84473673502604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,32,4,128,0,1,float16,float16,0,59.78248596191406
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,32,1,128,0,1,float16,float16,0,29.858133951822918
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16384,32,1,128,0,1,float16,float16,0,118.3970947265625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16384,32,2,128,0,1,float16,float16,0,117.28964233398438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,32,32,128,0,1,float16,float16,0,58.80200703938802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,32,32,128,0,1,float16,float16,0,29.328043619791668
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16384,32,4,128,0,1,float16,float16,0,118.64490763346355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,32,8,128,0,1,float16,float16,0,59.15272521972656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,32,2,128,0,1,float16,float16,0,29.857279459635418
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,32,4,128,0,1,float16,float16,0,29.862398783365887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,32,1,128,0,1,float16,float16,0,14.945962270100912
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16384,32,8,128,0,1,float16,float16,0,118.7578837076823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,32,2,128,0,1,float16,float16,0,15.294976552327475
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,32,8,128,0,1,float16,float16,0,29.529769897460938
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,32,4,128,0,1,float16,float16,0,15.207082112630209
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,32,32,128,0,1,float16,float16,0,15.047850290934244
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,32,8,128,0,1,float16,float16,0,15.263231913248697
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,32,1,128,0,1,float16,float16,0,33.74677276611328
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,32,2,128,0,1,float16,float16,0,33.39417521158854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,32,4,128,0,1,float16,float16,0,33.850537618001304
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,12288,32,1,128,0,1,float16,float16,0,68.09378051757812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,12288,32,2,128,0,1,float16,float16,0,66.87692769368489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,32,1,128,0,1,float16,float16,0,17.108309427897137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,12288,32,4,128,0,1,float16,float16,0,67.96424357096355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,32,32,128,0,1,float16,float16,0,33.81350453694662
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,32,32,128,0,1,float16,float16,0,17.021439870198567
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,32,2,128,0,1,float16,float16,0,17.09294891357422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,32,8,128,0,1,float16,float16,0,33.816235860188804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,12288,32,8,128,0,1,float16,float16,0,68.06442769368489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,32,4,128,0,1,float16,float16,0,16.682154337565105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,32,1,128,0,1,float16,float16,0,8.622933069864908
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,32,2,128,0,1,float16,float16,0,8.570538838704428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,32,32,128,0,1,float16,float16,0,8.855722427368164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,32,8,128,0,1,float16,float16,0,17.140907287597656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,32,4,128,0,1,float16,float16,0,8.721749623616537
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,32,8,128,0,1,float16,float16,0,8.633514404296875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,32,1,128,0,1,float16,float16,0,23.74536641438802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,32,2,128,0,1,float16,float16,0,23.7653325398763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,32,4,128,0,1,float16,float16,0,23.81243642171224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,10240,32,1,128,0,1,float16,float16,0,47.52315775553385
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,10240,32,2,128,0,1,float16,float16,0,46.45683288574219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,10240,32,4,128,0,1,float16,float16,0,46.67272440592448
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,32,1,128,0,1,float16,float16,0,11.956223805745443
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,32,32,128,0,1,float16,float16,0,12.218709309895834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,32,8,128,0,1,float16,float16,0,23.82080078125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,32,32,128,0,1,float16,float16,0,24.076629638671875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,10240,32,8,128,0,1,float16,float16,0,46.7220458984375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,32,4,128,0,1,float16,float16,0,11.79101816813151
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,32,2,128,0,1,float16,float16,0,12.059135437011719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,32,1,128,0,1,float16,float16,0,6.170453389485677
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,32,2,128,0,1,float16,float16,0,6.147925059000651
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,32,8,128,0,1,float16,float16,0,11.902122497558594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,32,32,128,0,1,float16,float16,0,6.194175720214844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,32,4,128,0,1,float16,float16,0,6.150997161865234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,32,8,128,0,1,float16,float16,0,6.090922673543294
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,32,1,128,0,1,float16,float16,0,30.66675313313802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,32,2,128,0,1,float16,float16,0,30.67187245686849
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,32,4,128,0,1,float16,float16,0,30.38958994547526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,32,1,128,0,1,float16,float16,0,15.17806879679362
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,8192,32,1,128,0,1,float16,float16,0,60.53871154785156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,8192,32,2,128,0,1,float16,float16,0,61.718526204427086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,8192,32,4,128,0,1,float16,float16,0,62.25902811686198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,32,32,128,0,1,float16,float16,0,15.686485290527344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,32,8,128,0,1,float16,float16,0,30.86182403564453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,32,2,128,0,1,float16,float16,0,15.12942886352539
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,32,32,128,0,1,float16,float16,0,31.086591084798176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,32,4,128,0,1,float16,float16,0,15.022079467773438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,32,1,128,0,1,float16,float16,0,7.883093516031901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,8192,32,8,128,0,1,float16,float16,0,61.99278767903646
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,32,8,128,0,1,float16,float16,0,15.585280100504557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,32,2,128,0,1,float16,float16,0,7.789397557576497
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,32,4,128,0,1,float16,float16,0,7.661567687988281
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,32,32,128,0,1,float16,float16,0,7.959210713704427
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,32,32,128,0,1,float16,float16,0,3.9855785369873047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,32,8,128,0,1,float16,float16,0,7.725738525390625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,32,1,128,0,1,float16,float16,0,4.003157297770183
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,32,4,128,0,1,float16,float16,0,3.9970134099324546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,32,2,128,0,1,float16,float16,0,3.998719851175944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,32,8,128,0,1,float16,float16,0,4.001109441121419
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,32,2,128,0,1,float16,float16,0,17.770837148030598
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,32,1,128,0,1,float16,float16,0,17.730047861735027
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,32,4,128,0,1,float16,float16,0,17.62338129679362
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,6144,32,1,128,0,1,float16,float16,0,34.85542297363281
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,6144,32,2,128,0,1,float16,float16,0,35.43603261311849
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,32,1,128,0,1,float16,float16,0,8.887637456258139
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,6144,32,4,128,0,1,float16,float16,0,34.60881042480469
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,32,2,128,0,1,float16,float16,0,8.685738881429037
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,32,32,128,0,1,float16,float16,0,18.07274627685547
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,6144,32,8,128,0,1,float16,float16,0,35.09503936767578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,32,8,128,0,1,float16,float16,0,17.836031595865887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,32,4,128,0,1,float16,float16,0,8.921429316202799
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,32,1,128,0,1,float16,float16,0,4.404394785563151
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,32,32,128,0,1,float16,float16,0,9.144320170084635
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,32,8,128,0,1,float16,float16,0,8.892245610555014
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,32,1,128,0,1,float16,float16,0,2.343594710032145
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,32,2,128,0,1,float16,float16,0,4.468906720479329
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,32,4,128,0,1,float16,float16,0,4.45030403137207
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,32,2,128,0,1,float16,float16,0,2.3470080693562827
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,32,32,128,0,1,float16,float16,0,4.462592124938965
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,32,32,128,0,1,float16,float16,0,2.3504212697347007
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,32,8,128,0,1,float16,float16,0,4.5102081298828125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,32,4,128,0,1,float16,float16,0,2.360149383544922
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,32,8,128,0,1,float16,float16,0,2.3459839820861816
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,32,1,128,0,1,float16,float16,0,16.46933364868164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,32,2,128,0,1,float16,float16,0,16.39406967163086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,32,4,128,0,1,float16,float16,0,16.32512029012044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,32,1,128,0,1,float16,float16,0,8.061610539754232
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,4096,32,1,128,0,1,float16,float16,0,32.79052734375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,4096,32,2,128,0,1,float16,float16,0,32.63880411783854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,4096,32,4,128,0,1,float16,float16,0,33.001983642578125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,32,2,128,0,1,float16,float16,0,8.073898951212565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,32,8,128,0,1,float16,float16,0,16.620031992594402
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,32,32,128,0,1,float16,float16,0,17.014272054036457
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,32,32,128,0,1,float16,float16,0,8.487765630086264
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,32,4,128,0,1,float16,float16,0,8.266752243041992
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,4096,32,8,128,0,1,float16,float16,0,32.63300323486328
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,32,1,128,0,1,float16,float16,0,4.019029299418132
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,32,8,128,0,1,float16,float16,0,8.12390391031901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,32,2,128,0,1,float16,float16,0,4.116479873657227
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,32,4,128,0,1,float16,float16,0,4.084906578063965
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,32,8,128,0,1,float16,float16,0,4.123306592305501
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,32,1,128,0,1,float16,float16,0,2.102442741394043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,32,2,128,0,1,float16,float16,0,2.101418654123942
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,32,32,128,0,1,float16,float16,0,2.140672047932943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,32,4,128,0,1,float16,float16,0,2.1152426401774087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,32,32,128,0,1,float16,float16,0,4.231679916381836
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,32,1,128,0,1,float16,float16,0,1.1470506985982258
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,32,2,128,0,1,float16,float16,0,1.1419306596120198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,32,32,128,0,1,float16,float16,0,1.1535360018412273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,32,4,128,0,1,float16,float16,0,1.13919997215271
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,32,8,128,0,1,float16,float16,0,2.104661305745443
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,32,8,128,0,1,float16,float16,0,1.1453440189361572
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,32,1,128,0,1,float16,float16,0,9.506986618041992
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,32,2,128,0,1,float16,float16,0,9.648640314737955
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,32,4,128,0,1,float16,float16,0,9.750357309977213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,3072,32,1,128,0,1,float16,float16,0,19.154261271158855
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,32,1,128,0,1,float16,float16,0,4.7503360112508135
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,3072,32,2,128,0,1,float16,float16,0,19.308202107747395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,3072,32,4,128,0,1,float16,float16,0,19.27526346842448
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,32,8,128,0,1,float16,float16,0,9.599658966064453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,32,32,128,0,1,float16,float16,0,10.103466669718424
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,32,2,128,0,1,float16,float16,0,4.696063995361328
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,32,32,128,0,1,float16,float16,0,5.029717445373535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,32,4,128,0,1,float16,float16,0,4.805802663167317
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,3072,32,8,128,0,1,float16,float16,0,19.191808064778645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,32,1,128,0,1,float16,float16,0,2.390357335408529
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,32,2,128,0,1,float16,float16,0,2.3934292793273926
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,32,4,128,0,1,float16,float16,0,2.400597254435221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,32,1,128,0,1,float16,float16,0,1.2579840024312336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,32,8,128,0,1,float16,float16,0,4.805290540059407
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,32,8,128,0,1,float16,float16,0,2.4123733838399253
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,32,2,128,0,1,float16,float16,0,1.2557653586069744
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,32,32,128,0,1,float16,float16,0,1.2948479652404785
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,32,4,128,0,1,float16,float16,0,1.2624213695526123
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,32,32,128,0,1,float16,float16,0,2.494976043701172
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,32,2,128,0,1,float16,float16,0,0.7062186400095621
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,32,1,128,0,1,float16,float16,0,0.705024003982544
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,32,8,128,0,1,float16,float16,0,1.264469305674235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,32,32,128,0,1,float16,float16,0,0.7069013118743896
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,32,4,128,0,1,float16,float16,0,0.7002453009287516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,32,8,128,0,1,float16,float16,0,0.7089493274688721
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,32,2,128,0,1,float16,float16,0,9.210538864135742
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,32,1,128,0,1,float16,float16,0,9.104554494222006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,32,4,128,0,1,float16,float16,0,9.446399688720703
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,32,1,128,0,1,float16,float16,0,4.533930778503418
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,2048,32,1,128,0,1,float16,float16,0,18.60488510131836
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,2048,32,2,128,0,1,float16,float16,0,18.704213460286457
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,2048,32,4,128,0,1,float16,float16,0,18.792959849039715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,32,2,128,0,1,float16,float16,0,4.572671890258789
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,32,8,128,0,1,float16,float16,0,9.28000005086263
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,32,4,128,0,1,float16,float16,0,4.587008158365886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,32,32,128,0,1,float16,float16,0,4.836693445841472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,32,32,128,0,1,float16,float16,0,9.762133280436197
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,2048,32,8,128,0,1,float16,float16,0,19.083093007405598
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,32,1,128,0,1,float16,float16,0,2.2570667266845703
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,32,1,128,0,1,float16,float16,0,1.1775999863942463
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,32,2,128,0,1,float16,float16,0,2.2596267064412436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,32,4,128,0,1,float16,float16,0,2.2662827173868814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,32,8,128,0,1,float16,float16,0,4.626773198445638
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,32,32,128,0,1,float16,float16,0,2.4297812779744468
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,32,2,128,0,1,float16,float16,0,1.1775999863942463
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,32,8,128,0,1,float16,float16,0,2.291541258494059
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,32,32,128,0,1,float16,float16,0,1.2417706648508708
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,32,1,128,0,1,float16,float16,0,0.6415359973907471
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,32,8,128,0,1,float16,float16,0,1.1864746411641438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,32,32,128,0,1,float16,float16,0,0.65228799978892
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,32,4,128,0,1,float16,float16,0,1.182037353515625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,32,2,128,0,1,float16,float16,0,0.6352213223775228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,32,4,128,0,1,float16,float16,0,0.6377813418706259
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,32,1,128,0,1,float16,float16,0,0.3679573138554891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,32,8,128,0,1,float16,float16,0,0.6396586497624716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,32,32,128,0,1,float16,float16,0,0.37085866928100586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,32,2,128,0,1,float16,float16,0,0.36471466223398846
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,32,4,128,0,1,float16,float16,0,0.36420265833536786
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,32,8,128,0,1,float16,float16,0,0.3696639935175578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,32,1,128,0,1,float16,float16,0,5.487103780110677
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,32,2,128,0,1,float16,float16,0,5.641045252482097
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,32,4,128,0,1,float16,float16,0,5.6248321533203125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1536,32,1,128,0,1,float16,float16,0,11.268778483072916
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1536,32,2,128,0,1,float16,float16,0,11.382101694742838
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,32,1,128,0,1,float16,float16,0,2.7067734400431314
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1536,32,4,128,0,1,float16,float16,0,11.336533864339193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,32,2,128,0,1,float16,float16,0,2.7246932983398438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,32,8,128,0,1,float16,float16,0,5.609642664591472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,32,32,128,0,1,float16,float16,0,2.9832534790039062
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,32,32,128,0,1,float16,float16,0,6.023679733276367
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,32,4,128,0,1,float16,float16,0,2.73527463277181
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,32,1,128,0,1,float16,float16,0,1.3791573842366536
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,32,2,128,0,1,float16,float16,0,1.382570743560791
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1536,32,8,128,0,1,float16,float16,0,11.333802541097006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,32,4,128,0,1,float16,float16,0,1.3911040623982747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,32,8,128,0,1,float16,float16,0,2.748074531555176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,32,32,128,0,1,float16,float16,0,1.5076692899068196
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,32,8,128,0,1,float16,float16,0,1.399295965830485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,32,1,128,0,1,float16,float16,0,0.7284053166707357
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,32,2,128,0,1,float16,float16,0,0.7284053166707357
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,32,4,128,0,1,float16,float16,0,0.7309652964274088
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,32,8,128,0,1,float16,float16,0,0.7323306401570638
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,32,1,128,0,1,float16,float16,0,0.40174933274586994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,32,2,128,0,1,float16,float16,0,0.3998719851175944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,32,32,128,0,1,float16,float16,0,0.7741440137227377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,32,4,128,0,1,float16,float16,0,0.40345601240793866
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,32,8,128,0,1,float16,float16,0,0.40635732809702557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,32,32,128,0,1,float16,float16,0,0.41710933049519855
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,32,1,128,0,1,float16,float16,0,0.2373973329861959
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,32,32,128,0,1,float16,float16,0,0.24371200799942017
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,32,2,128,0,1,float16,float16,0,0.23893332481384277
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,32,4,128,0,1,float16,float16,0,0.23637332518895468
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,32,8,128,0,1,float16,float16,0,0.2379093368848165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,32,1,128,0,1,float16,float16,0,5.386240005493164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,32,2,128,0,1,float16,float16,0,5.3567148844401045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,32,4,128,0,1,float16,float16,0,5.5137278238932295
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,1024,32,1,128,0,1,float16,float16,0,10.975232442220053
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,32,1,128,0,1,float16,float16,0,2.668544133504232
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,1024,32,2,128,0,1,float16,float16,0,10.809856414794922
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,1024,32,4,128,0,1,float16,float16,0,11.199317932128906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,32,2,128,0,1,float16,float16,0,2.676394780476888
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,32,8,128,0,1,float16,float16,0,5.564074834187825
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,32,4,128,0,1,float16,float16,0,2.7089920043945312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,1024,32,8,128,0,1,float16,float16,0,11.332436879475912
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,32,32,128,0,1,float16,float16,0,3.1225172678629556
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,32,32,128,0,1,float16,float16,0,6.273877461751302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,32,8,128,0,1,float16,float16,0,2.742442766825358
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,32,1,128,0,1,float16,float16,0,1.3521919250488281
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,32,2,128,0,1,float16,float16,0,1.3641386032104492
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,32,4,128,0,1,float16,float16,0,1.3702826499938965
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,32,1,128,0,1,float16,float16,0,0.6990506649017334
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,32,2,128,0,1,float16,float16,0,0.6981973648071289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,32,4,128,0,1,float16,float16,0,0.7029759883880615
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,32,8,128,0,1,float16,float16,0,1.390933354695638
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,32,32,128,0,1,float16,float16,0,0.8034986654917399
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,32,32,128,0,1,float16,float16,0,1.5774720509847004
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,32,1,128,0,1,float16,float16,0,0.36829864978790283
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,32,8,128,0,1,float16,float16,0,0.7150932947794596
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,32,2,128,0,1,float16,float16,0,0.36829864978790283
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,32,4,128,0,1,float16,float16,0,0.3737599849700928
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,32,32,128,0,1,float16,float16,0,0.4159146547317505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,32,8,128,0,1,float16,float16,0,0.37597866853078205
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,32,1,128,0,1,float16,float16,0,0.20258132616678873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,32,2,128,0,1,float16,float16,0,0.20121600230534872
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,32,32,128,0,1,float16,float16,0,0.22818134228388467
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,32,4,128,0,1,float16,float16,0,0.2039466698964437
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,32,1,128,0,1,float16,float16,0,0.11997866630554199
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,32,8,128,0,1,float16,float16,0,0.2065066695213318
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,32,4,128,0,1,float16,float16,0,0.11741866668065389
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,32,32,128,0,1,float16,float16,0,0.13056000073750815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,32,2,128,0,1,float16,float16,0,0.11707733074824016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,32,8,128,0,1,float16,float16,0,0.11758933464686076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,32,1,128,0,1,float16,float16,0,3.6225706736246743
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,32,2,128,0,1,float16,float16,0,3.6439040501912436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,32,4,128,0,1,float16,float16,0,3.7200212478637695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,512,32,1,128,0,1,float16,float16,0,7.296853383382161
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,32,1,128,0,1,float16,float16,0,1.8140160242716472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,512,32,2,128,0,1,float16,float16,0,7.3560746510823565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,512,32,4,128,0,1,float16,float16,0,7.609343846638997
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,32,32,128,0,1,float16,float16,0,2.276522636413574
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,32,2,128,0,1,float16,float16,0,1.8257919947306316
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,32,8,128,0,1,float16,float16,0,3.804501215616862
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,32,4,128,0,1,float16,float16,0,1.850538730621338
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,32,32,128,0,1,float16,float16,0,4.570282618204753
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,512,32,8,128,0,1,float16,float16,0,7.6885331471761065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,32,1,128,0,1,float16,float16,0,0.9224533240000407
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,32,8,128,0,1,float16,float16,0,1.9017386436462402
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,32,2,128,0,1,float16,float16,0,0.9238186677296957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,32,1,128,0,1,float16,float16,0,0.4761600097020467
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,32,4,128,0,1,float16,float16,0,0.936959981918335
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,32,2,128,0,1,float16,float16,0,0.47308798631032306
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,32,32,128,0,1,float16,float16,0,1.1530240376790364
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,32,4,128,0,1,float16,float16,0,0.48401065667470294
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,32,8,128,0,1,float16,float16,0,0.9567573070526123
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,32,32,128,0,1,float16,float16,0,0.5847040017445883
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,32,8,128,0,1,float16,float16,0,0.4927146832148234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,32,32,128,0,1,float16,float16,0,0.30190932750701904
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,32,1,128,0,1,float16,float16,0,0.25173334280649823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,32,2,128,0,1,float16,float16,0,0.2512213389078776
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,32,4,128,0,1,float16,float16,0,0.2573653260866801
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,32,1,128,0,1,float16,float16,0,0.1372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,32,8,128,0,1,float16,float16,0,0.2606079975763957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,32,32,128,0,1,float16,float16,0,0.16554666558901468
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,32,2,128,0,1,float16,float16,0,0.13858133554458618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,32,4,128,0,1,float16,float16,0,0.13926399747530618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,32,8,128,0,1,float16,float16,0,0.14216533303260803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,32,32,128,0,1,float16,float16,0,0.09318400422732036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,32,1,128,0,1,float16,float16,0,0.08021333316961925
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,32,4,128,0,1,float16,float16,0,0.08004266520341237
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,32,8,128,0,1,float16,float16,0,0.08072533210118611
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,32,32,128,0,1,float16,float16,0,0.055125330885251365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,32,1,128,0,1,float16,float16,0,0.053930665055910744
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,32,2,128,0,1,float16,float16,0,0.07935999830563863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,32,4,128,0,1,float16,float16,0,0.054272000988324486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,32,8,128,0,1,float16,float16,0,0.05341866612434387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,32,2,128,0,1,float16,float16,0,0.05256533126036326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,32,1,128,0,1,float16,float16,0,1.3789866765340169
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,32,2,128,0,1,float16,float16,0,1.3801813125610352
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,32,4,128,0,1,float16,float16,0,1.4269439379374187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,256,32,1,128,0,1,float16,float16,0,2.7685546875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,256,32,2,128,0,1,float16,float16,0,2.757802645365397
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,256,32,4,128,0,1,float16,float16,0,2.8769280115763345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,32,1,128,0,1,float16,float16,0,0.7019519805908203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,32,8,128,0,1,float16,float16,0,1.4646612803141277
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,256,32,8,128,0,1,float16,float16,0,2.956629435221354
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,32,32,128,0,1,float16,float16,0,1.8737492561340332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,32,2,128,0,1,float16,float16,0,0.7043413321177164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,32,32,128,0,1,float16,float16,0,0.9501012961069742
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,32,4,128,0,1,float16,float16,0,0.7226026852925619
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,32,1,128,0,1,float16,float16,0,0.36266668637593585
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,32,2,128,0,1,float16,float16,0,0.3633493185043335
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,32,8,128,0,1,float16,float16,0,0.7441066900889078
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,32,1,128,0,1,float16,float16,0,0.19165867567062378
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,32,4,128,0,1,float16,float16,0,0.3715413411458333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,32,8,128,0,1,float16,float16,0,0.3809279998143514
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,32,32,128,0,1,float16,float16,0,0.26129066944122314
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,32,32,128,0,1,float16,float16,0,0.4877653519312541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,32,2,128,0,1,float16,float16,0,0.19131733973821005
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,32,4,128,0,1,float16,float16,0,0.19524266322453818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,32,1,128,0,1,float16,float16,0,0.10444800059000652
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,32,8,128,0,1,float16,float16,0,0.2027519941329956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,32,2,128,0,1,float16,float16,0,0.10496000448862712
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,32,4,128,0,1,float16,float16,0,0.10598400235176086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,32,32,128,0,1,float16,float16,0,0.1397760013739268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,32,32,128,0,1,float16,float16,0,0.07543466488520305
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,32,1,128,0,1,float16,float16,0,0.06126933296521505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,32,8,128,0,1,float16,float16,0,0.10990933577219646
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,32,2,128,0,1,float16,float16,0,0.06109866499900818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,32,4,128,0,1,float16,float16,0,0.061610668897628784
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,32,1,128,0,1,float16,float16,0,0.04027733455101649
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,32,8,128,0,1,float16,float16,0,0.06297599772612254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,32,32,128,0,1,float16,float16,0,0.04181333382924398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,32,2,128,0,1,float16,float16,0,0.040106666584809623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,32,4,128,0,1,float16,float16,0,0.040448000033696495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,32,32,128,0,1,float16,float16,0,0.0314026673634847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,32,1,128,0,1,float16,float16,0,0.029696000119050343
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,32,2,128,0,1,float16,float16,0,0.029696000119050343
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,32,8,128,0,1,float16,float16,0,0.039936001102129616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,32,8,128,0,1,float16,float16,0,0.030037333567937214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,32,4,128,0,1,float16,float16,0,0.030037333567937214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,32,2,128,0,1,float16,float16,0,0.6026240189870199
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,32,1,128,0,1,float16,float16,0,0.5986986557642618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,32,4,128,0,1,float16,float16,0,0.6217386722564697
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,128,32,1,128,0,1,float16,float16,0,1.180842638015747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,128,32,2,128,0,1,float16,float16,0,1.1859626770019531
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,32,1,128,0,1,float16,float16,0,0.3264853358268738
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,32,2,128,0,1,float16,float16,0,0.31214932600657147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,128,32,4,128,0,1,float16,float16,0,1.237162669499715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,32,4,128,0,1,float16,float16,0,0.32153600454330444
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,128,32,8,128,0,1,float16,float16,0,1.2840960025787354
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,32,32,128,0,1,float16,float16,0,0.9081172943115234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,32,8,128,0,1,float16,float16,0,0.6463146607081095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,32,1,128,0,1,float16,float16,0,0.1629866659641266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,32,32,128,0,1,float16,float16,0,0.4643839995066325
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,32,2,128,0,1,float16,float16,0,0.16315733393033346
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,32,8,128,0,1,float16,float16,0,0.3322880069414775
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,32,4,128,0,1,float16,float16,0,0.169813334941864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,32,32,128,0,1,float16,float16,0,0.24524799982706705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,32,8,128,0,1,float16,float16,0,0.1764693260192871
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,32,32,128,0,1,float16,float16,0,0.13380266229311624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,32,1,128,0,1,float16,float16,0,0.08669867118199666
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,32,1,128,0,1,float16,float16,0,0.04983466863632202
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,32,4,128,0,1,float16,float16,0,0.08994133273760478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,32,8,128,0,1,float16,float16,0,0.09454933802286784
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,32,2,128,0,1,float16,float16,0,0.05017599960168203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,32,32,128,0,1,float16,float16,0,0.06758399804433186
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,32,4,128,0,1,float16,float16,0,0.04949333270390829
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,32,2,128,0,1,float16,float16,0,0.08789333701133728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,32,8,128,0,1,float16,float16,0,0.050517335534095764
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,32,1,128,0,1,float16,float16,0,0.034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,32,2,128,0,1,float16,float16,0,0.0339626669883728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,32,4,128,0,1,float16,float16,0,0.034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,32,32,128,0,1,float16,float16,0,0.03601066768169403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,32,8,128,0,1,float16,float16,0,0.0341333324710528
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,32,32,128,0,1,float16,float16,0,0.025600001215934753
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,32,2,128,0,1,float16,float16,0,0.024234667420387268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,32,4,128,0,1,float16,float16,0,0.0240639994541804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,32,8,128,0,1,float16,float16,0,0.024405332903067272
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,32,2,128,0,1,float16,float16,0,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,32,32,128,0,1,float16,float16,0,0.01877333347996076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,32,1,128,0,1,float16,float16,0,0.01877333347996076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,32,1,128,0,1,float16,float16,0,0.0240639994541804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,32,4,128,0,1,float16,float16,0,0.018432000031073887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,32,8,128,0,1,float16,float16,0,0.018432000031073887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,32,2,128,0,1,float16,float16,0,0.30754133065541583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,32,1,128,0,1,float16,float16,0,0.3031040032704671
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,32,4,128,0,1,float16,float16,0,0.31948800881703693
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,64,32,1,128,0,1,float16,float16,0,0.5816320180892944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,64,32,2,128,0,1,float16,float16,0,0.5932373205820719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,64,32,4,128,0,1,float16,float16,0,0.6176426808039347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,32,32,128,0,1,float16,float16,0,0.4664320151011149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,64,32,8,128,0,1,float16,float16,0,0.6473386685053507
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,32,1,128,0,1,float16,float16,0,0.1634986698627472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,32,2,128,0,1,float16,float16,0,0.1807360053062439
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,32,4,128,0,1,float16,float16,0,0.1728853384653727
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,32,8,128,0,1,float16,float16,0,0.33774932225545246
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,32,8,128,0,1,float16,float16,0,0.18227199713389078
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,32,1,128,0,1,float16,float16,0,0.08618666728337605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,32,32,128,0,1,float16,float16,0,0.13363200426101685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,32,2,128,0,1,float16,float16,0,0.08721066514650981
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,32,4,128,0,1,float16,float16,0,0.09130666653315227
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,32,32,128,0,1,float16,float16,0,0.24422399202982584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,32,8,128,0,1,float16,float16,0,0.09608532985051473
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,32,32,128,0,1,float16,float16,0,0.06758399804433186
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,32,1,128,0,1,float16,float16,0,0.05000533163547516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,32,2,128,0,1,float16,float16,0,0.05017599960168203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,32,4,128,0,1,float16,float16,0,0.0506879985332489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,32,8,128,0,1,float16,float16,0,0.05239466826121012
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,32,32,128,0,1,float16,float16,0,0.037205333511034645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,32,4,128,0,1,float16,float16,0,0.034645333886146545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,32,2,128,0,1,float16,float16,0,0.034815999368826546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,32,1,128,0,1,float16,float16,0,0.034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,32,8,128,0,1,float16,float16,0,0.034815999368826546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,32,32,128,0,1,float16,float16,0,0.024405332903067272
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,32,1,128,0,1,float16,float16,0,0.023381332556406658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,32,2,128,0,1,float16,float16,0,0.023381332556406658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,32,4,128,0,1,float16,float16,0,0.023381332556406658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,32,8,128,0,1,float16,float16,0,0.023381332556406658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,32,32,128,0,1,float16,float16,0,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,32,2,128,0,1,float16,float16,0,0.01791999985774358
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,32,4,128,0,1,float16,float16,0,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,32,8,128,0,1,float16,float16,0,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,32,32,128,0,1,float16,float16,0,0.016895999511082966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,32,1,128,0,1,float16,float16,0,0.0170666662355264
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,32,1,128,0,1,float16,float16,0,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,32,2,128,0,1,float16,float16,0,0.0170666662355264
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,32,4,128,0,1,float16,float16,0,0.0170666662355264
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,32,8,128,0,1,float16,float16,0,0.016895999511082966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,32,32,1,128,0,1,float16,float16,0,0.34406399726867676
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,32,32,2,128,0,1,float16,float16,0,0.3471359809239705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,32,1,128,0,1,float16,float16,0,0.1790293256441752
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,32,2,128,0,1,float16,float16,0,0.18193066120147705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,32,4,128,0,1,float16,float16,0,0.1853440006573995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,32,32,4,128,0,1,float16,float16,0,0.3529386520385742
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,32,32,8,128,0,1,float16,float16,0,0.36454399426778156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,32,1,128,0,1,float16,float16,0,0.0936959981918335
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,32,2,128,0,1,float16,float16,0,0.09489066402117412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,32,32,128,0,1,float16,float16,0,0.250709335009257
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,32,32,128,0,1,float16,float16,0,0.13175466656684875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,32,4,128,0,1,float16,float16,0,0.09864532947540283
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,32,8,128,0,1,float16,float16,0,0.1914880077044169
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,32,8,128,0,1,float16,float16,0,0.10240000486373901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,32,32,128,0,1,float16,float16,0,0.06911999980608623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,32,1,128,0,1,float16,float16,0,0.051370665431022644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,32,2,128,0,1,float16,float16,0,0.051882664362589516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,32,1,128,0,1,float16,float16,0,0.0339626669883728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,32,2,128,0,1,float16,float16,0,0.0341333324710528
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,32,8,128,0,1,float16,float16,0,0.053247998158137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,32,32,128,0,1,float16,float16,0,0.03549866626660029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,32,4,128,0,1,float16,float16,0,0.051882664362589516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,32,8,128,0,1,float16,float16,0,0.034304000437259674
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,32,4,128,0,1,float16,float16,0,0.03379199902216593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,32,1,128,0,1,float16,float16,0,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,32,32,128,0,1,float16,float16,0,0.02491733431816101
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,32,2,128,0,1,float16,float16,0,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,32,4,128,0,1,float16,float16,0,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,32,8,128,0,1,float16,float16,0,0.024746666351954143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,32,1,128,0,1,float16,float16,0,0.0170666662355264
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,32,4,128,0,1,float16,float16,0,0.017237332959969837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,32,8,128,0,1,float16,float16,0,0.017237332959969837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,32,2,128,0,1,float16,float16,0,0.0170666662355264
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,32,32,128,0,1,float16,float16,0,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,32,1,128,0,1,float16,float16,0,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,32,2,128,0,1,float16,float16,0,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,32,32,128,0,1,float16,float16,0,0.0170666662355264
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,32,4,128,0,1,float16,float16,0,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,32,8,128,0,1,float16,float16,0,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,32,1,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,32,32,128,0,1,float16,float16,0,0.013823999712864557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,32,4,128,0,1,float16,float16,0,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,32,2,128,0,1,float16,float16,0,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,32,8,128,0,1,float16,float16,0,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,16,32,2,128,0,1,float16,float16,0,0.2698240081469218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,32,1,128,0,1,float16,float16,0,0.1346560021241506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,32,4,128,0,1,float16,float16,0,0.1397760013739268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,32,2,128,0,1,float16,float16,0,0.13619200388590494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,16,32,1,128,0,1,float16,float16,0,0.2672640085220337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,16,32,4,128,0,1,float16,float16,0,0.27426133553187054
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,32,32,128,0,1,float16,float16,0,0.1621333360671997
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,16,32,8,128,0,1,float16,float16,0,0.2802346746126811
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,32,1,128,0,1,float16,float16,0,0.0721919983625412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,32,2,128,0,1,float16,float16,0,0.07236266632874806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,32,8,128,0,1,float16,float16,0,0.14472533265749613
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,32,32,128,0,1,float16,float16,0,0.08516266942024231
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,32,1,128,0,1,float16,float16,0,0.04232533276081085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,32,32,128,0,1,float16,float16,0,0.043178667624791466
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,32,8,128,0,1,float16,float16,0,0.07441066702206929
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,32,4,128,0,1,float16,float16,0,0.07253333429495494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,32,2,128,0,1,float16,float16,0,0.04249600072701772
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,32,8,128,0,1,float16,float16,0,0.042837331692377724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,32,4,128,0,1,float16,float16,0,0.04249600072701772
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,32,1,128,0,1,float16,float16,0,0.0290133332212766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,32,2,128,0,1,float16,float16,0,0.028501334289709728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,32,32,128,0,1,float16,float16,0,0.02867199977238973
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,32,4,128,0,1,float16,float16,0,0.02867199977238973
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,32,32,128,0,1,float16,float16,0,0.02065066620707512
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,32,1,128,0,1,float16,float16,0,0.021333334346612293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,32,2,128,0,1,float16,float16,0,0.020821332931518555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,32,8,128,0,1,float16,float16,0,0.02065066620707512
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,32,4,128,0,1,float16,float16,0,0.021674667795499165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,32,32,128,0,1,float16,float16,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,32,8,128,0,1,float16,float16,0,0.0290133332212766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,32,2,128,0,1,float16,float16,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,32,4,128,0,1,float16,float16,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,32,1,128,0,1,float16,float16,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,32,1,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,32,8,128,0,1,float16,float16,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,32,32,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,32,2,128,0,1,float16,float16,0,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,32,4,128,0,1,float16,float16,0,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,32,8,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,32,32,128,0,1,float16,float16,0,0.013653332988421122
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,32,2,128,0,1,float16,float16,0,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,32,4,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,32,1,128,0,1,float16,float16,0,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,32,8,128,0,1,float16,float16,0,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,24,1,128,0,1,float16,float16,0,44.39603169759115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,24,2,128,0,1,float16,float16,0,43.6140391031901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,24,4,128,0,1,float16,float16,0,44.38067118326823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,24,1,128,0,1,float16,float16,0,22.239913940429688
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16384,24,1,128,0,1,float16,float16,0,87.7337646484375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16384,24,2,128,0,1,float16,float16,0,89.7780049641927
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,24,24,128,0,1,float16,float16,0,44.744873046875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16384,24,4,128,0,1,float16,float16,0,88.58418782552083
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,24,24,128,0,1,float16,float16,0,22.015487670898438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,24,2,128,0,1,float16,float16,0,22.264490763346355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,24,8,128,0,1,float16,float16,0,44.651346842447914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,24,4,128,0,1,float16,float16,0,22.056106567382812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,24,1,128,0,1,float16,float16,0,11.295232137044271
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,24,2,128,0,1,float16,float16,0,11.24676259358724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16384,24,8,128,0,1,float16,float16,0,89.6730448404948
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,24,24,128,0,1,float16,float16,0,11.602261861165365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,24,4,128,0,1,float16,float16,0,11.52358373006185
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,24,8,128,0,1,float16,float16,0,22.54028828938802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,24,8,128,0,1,float16,float16,0,11.537066141764322
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,24,1,128,0,1,float16,float16,0,25.40851084391276
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,24,2,128,0,1,float16,float16,0,25.444351196289062
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,24,4,128,0,1,float16,float16,0,25.411582946777344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,12288,24,1,128,0,1,float16,float16,0,49.77100626627604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,12288,24,2,128,0,1,float16,float16,0,50.00209045410156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,24,1,128,0,1,float16,float16,0,12.783616383870443
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,24,24,128,0,1,float16,float16,0,25.729876200358074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,24,2,128,0,1,float16,float16,0,12.6112429300944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,12288,24,4,128,0,1,float16,float16,0,49.29706827799479
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,24,24,128,0,1,float16,float16,0,13.059242248535156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,24,4,128,0,1,float16,float16,0,12.696404774983725
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,24,8,128,0,1,float16,float16,0,25.342122395833332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,24,1,128,0,1,float16,float16,0,6.544042587280273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,24,2,128,0,1,float16,float16,0,6.4076798756917315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,12288,24,8,128,0,1,float16,float16,0,50.83153279622396
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,24,8,128,0,1,float16,float16,0,12.802730560302734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,24,4,128,0,1,float16,float16,0,6.5831254323323565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,24,24,128,0,1,float16,float16,0,6.640127817789714
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,24,8,128,0,1,float16,float16,0,6.51963742574056
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,24,1,128,0,1,float16,float16,0,17.897472381591797
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,24,2,128,0,1,float16,float16,0,17.864704132080078
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,24,4,128,0,1,float16,float16,0,17.678677876790363
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,10240,24,1,128,0,1,float16,float16,0,35.38312530517578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,10240,24,2,128,0,1,float16,float16,0,35.03394063313802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,10240,24,4,128,0,1,float16,float16,0,34.65403747558594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,24,1,128,0,1,float16,float16,0,8.897194544474283
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,24,24,128,0,1,float16,float16,0,18.129578908284504
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,24,2,128,0,1,float16,float16,0,8.917845408121744
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,24,24,128,0,1,float16,float16,0,9.190570831298828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,24,4,128,0,1,float16,float16,0,8.914431889851889
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,10240,24,8,128,0,1,float16,float16,0,35.52016957600912
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,24,8,128,0,1,float16,float16,0,17.95242691040039
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,24,1,128,0,1,float16,float16,0,4.662613232930501
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,24,2,128,0,1,float16,float16,0,4.643157323201497
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,24,4,128,0,1,float16,float16,0,4.635135968526204
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,24,24,128,0,1,float16,float16,0,4.596735954284668
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,24,8,128,0,1,float16,float16,0,8.929962793986002
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,24,8,128,0,1,float16,float16,0,4.572160085042317
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,24,1,128,0,1,float16,float16,0,22.790143330891926
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,24,2,128,0,1,float16,float16,0,22.58227284749349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,24,4,128,0,1,float16,float16,0,22.698496500651043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,8192,24,1,128,0,1,float16,float16,0,45.96855672200521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,24,1,128,0,1,float16,float16,0,11.423402150472006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,8192,24,2,128,0,1,float16,float16,0,46.258687337239586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,8192,24,4,128,0,1,float16,float16,0,46.14263407389323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,24,2,128,0,1,float16,float16,0,11.6495361328125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,24,24,128,0,1,float16,float16,0,23.419904073079426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,24,8,128,0,1,float16,float16,0,22.887593587239582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,24,24,128,0,1,float16,float16,0,11.546112060546875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,24,4,128,0,1,float16,float16,0,11.404287974039713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,24,1,128,0,1,float16,float16,0,5.874517440795898
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,24,2,128,0,1,float16,float16,0,5.882709503173828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,24,8,128,0,1,float16,float16,0,11.734869639078775
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,24,24,128,0,1,float16,float16,0,5.975210825602214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,24,4,128,0,1,float16,float16,0,5.840213139851888
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,8192,24,8,128,0,1,float16,float16,0,46.09962463378906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,24,8,128,0,1,float16,float16,0,5.741909027099609
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,24,1,128,0,1,float16,float16,0,3.0277973810831704
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,24,2,128,0,1,float16,float16,0,3.036160151163737
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,24,4,128,0,1,float16,float16,0,3.0407679875691733
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,24,24,128,0,1,float16,float16,0,3.031381289164225
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,24,8,128,0,1,float16,float16,0,3.0419626235961914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,24,1,128,0,1,float16,float16,0,13.331456502278646
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,24,2,128,0,1,float16,float16,0,13.168980916341146
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,24,4,128,0,1,float16,float16,0,13.343402862548828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,6144,24,1,128,0,1,float16,float16,0,26.17821756998698
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,24,1,128,0,1,float16,float16,0,6.670677185058594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,6144,24,2,128,0,1,float16,float16,0,26.525014241536457
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,24,24,128,0,1,float16,float16,0,13.642069498697916
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,6144,24,4,128,0,1,float16,float16,0,25.970687866210938
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,24,2,128,0,1,float16,float16,0,6.587562561035156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,24,24,128,0,1,float16,float16,0,6.718463897705078
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,24,8,128,0,1,float16,float16,0,13.38760503133138
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,24,4,128,0,1,float16,float16,0,6.689621607462565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,24,1,128,0,1,float16,float16,0,3.3327786127726235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,6144,24,8,128,0,1,float16,float16,0,26.678443908691406
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,24,2,128,0,1,float16,float16,0,3.355306625366211
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,24,8,128,0,1,float16,float16,0,6.69593620300293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,24,4,128,0,1,float16,float16,0,3.3495041529337564
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,24,24,128,0,1,float16,float16,0,3.3976319630940757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,24,8,128,0,1,float16,float16,0,3.3802239100138345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,24,1,128,0,1,float16,float16,0,1.7949013710021973
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,24,2,128,0,1,float16,float16,0,1.8003627459208171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,24,24,128,0,1,float16,float16,0,1.8114560445149739
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,24,4,128,0,1,float16,float16,0,1.793706734975179
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,24,8,128,0,1,float16,float16,0,1.803434689839681
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,24,1,128,0,1,float16,float16,0,12.36343510945638
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,24,2,128,0,1,float16,float16,0,12.347391764322916
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,24,4,128,0,1,float16,float16,0,12.295338948567709
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,4096,24,1,128,0,1,float16,float16,0,24.372395833333332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,24,1,128,0,1,float16,float16,0,6.1235198974609375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,4096,24,2,128,0,1,float16,float16,0,24.77056121826172
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,4096,24,4,128,0,1,float16,float16,0,24.811350504557293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,24,2,128,0,1,float16,float16,0,6.042794545491536
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,24,24,128,0,1,float16,float16,0,6.314154942830403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,24,8,128,0,1,float16,float16,0,12.573354085286459
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,24,24,128,0,1,float16,float16,0,12.801024119059244
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,24,1,128,0,1,float16,float16,0,3.037013371785482
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,24,4,128,0,1,float16,float16,0,6.143829345703125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,24,8,128,0,1,float16,float16,0,6.242133458455403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,4096,24,8,128,0,1,float16,float16,0,24.4312744140625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,24,2,128,0,1,float16,float16,0,3.0407679875691733
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,24,4,128,0,1,float16,float16,0,3.069098790486654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,24,1,128,0,1,float16,float16,0,1.6124587059020996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,24,24,128,0,1,float16,float16,0,3.1534080505371094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,24,2,128,0,1,float16,float16,0,1.5982933044433594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,24,24,128,0,1,float16,float16,0,1.6382293701171875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,24,8,128,0,1,float16,float16,0,3.0772905349731445
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,24,1,128,0,1,float16,float16,0,0.8845653533935547
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,24,4,128,0,1,float16,float16,0,1.5996586481730144
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,24,2,128,0,1,float16,float16,0,0.8869547049204508
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,24,8,128,0,1,float16,float16,0,1.607850710550944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,24,24,128,0,1,float16,float16,0,0.8905386924743652
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,24,4,128,0,1,float16,float16,0,0.8838826815287272
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,24,8,128,0,1,float16,float16,0,0.8884906768798828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,24,1,128,0,1,float16,float16,0,7.235925038655599
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,24,2,128,0,1,float16,float16,0,7.320917129516602
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,24,4,128,0,1,float16,float16,0,7.182336171468099
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,3072,24,1,128,0,1,float16,float16,0,14.52356211344401
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,3072,24,2,128,0,1,float16,float16,0,14.50222905476888
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,24,1,128,0,1,float16,float16,0,3.4950825373331704
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,3072,24,4,128,0,1,float16,float16,0,14.181888580322266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,24,24,128,0,1,float16,float16,0,3.740330696105957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,24,24,128,0,1,float16,float16,0,7.506090799967448
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,24,2,128,0,1,float16,float16,0,3.5520852406819663
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,24,8,128,0,1,float16,float16,0,7.323306401570638
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,3072,24,8,128,0,1,float16,float16,0,14.814549763997396
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,24,4,128,0,1,float16,float16,0,3.5577173233032227
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,24,8,128,0,1,float16,float16,0,3.5532798767089844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,24,1,128,0,1,float16,float16,0,1.8356906572977703
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,24,2,128,0,1,float16,float16,0,1.815893332163493
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,24,4,128,0,1,float16,float16,0,1.8227200508117676
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,24,8,128,0,1,float16,float16,0,1.837056001027425
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,24,1,128,0,1,float16,float16,0,0.9685333569844564
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,24,2,128,0,1,float16,float16,0,0.9669973055521647
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,24,4,128,0,1,float16,float16,0,0.9714346726735433
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,24,24,128,0,1,float16,float16,0,1.8950826327006023
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,24,24,128,0,1,float16,float16,0,0.9949866930643717
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,24,8,128,0,1,float16,float16,0,0.9731413523356119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,24,1,128,0,1,float16,float16,0,0.5495466788609823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,24,24,128,0,1,float16,float16,0,0.5589333375295004
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,24,4,128,0,1,float16,float16,0,0.5480106671651205
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,24,2,128,0,1,float16,float16,0,0.5481813351313273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,24,8,128,0,1,float16,float16,0,0.5534720023473104
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,24,1,128,0,1,float16,float16,0,6.891520182291667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,24,2,128,0,1,float16,float16,0,6.9998931884765625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,24,4,128,0,1,float16,float16,0,7.02020263671875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,2048,24,1,128,0,1,float16,float16,0,13.630804697672525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,24,1,128,0,1,float16,float16,0,3.366058667500814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,2048,24,2,128,0,1,float16,float16,0,14.050645192464193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,2048,24,4,128,0,1,float16,float16,0,13.938687642415365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,24,8,128,0,1,float16,float16,0,7.030954360961914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,24,2,128,0,1,float16,float16,0,3.379199981689453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,24,24,128,0,1,float16,float16,0,3.671210606892904
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,24,4,128,0,1,float16,float16,0,3.4097493489583335
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,24,24,128,0,1,float16,float16,0,7.449087778727214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,24,8,128,0,1,float16,float16,0,3.477333386739095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,2048,24,8,128,0,1,float16,float16,0,14.297088623046875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,24,1,128,0,1,float16,float16,0,1.7104214032491047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,24,2,128,0,1,float16,float16,0,1.7111040751139324
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,24,24,128,0,1,float16,float16,0,0.9524906476338705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,24,24,128,0,1,float16,float16,0,1.8399573961893718
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,24,4,128,0,1,float16,float16,0,1.7286826769510906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,24,1,128,0,1,float16,float16,0,0.8958293596903483
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,24,2,128,0,1,float16,float16,0,0.9021440347035726
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,24,4,128,0,1,float16,float16,0,0.8941226800282797
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,24,8,128,0,1,float16,float16,0,1.7384106318155925
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,24,1,128,0,1,float16,float16,0,0.4939093192418416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,24,8,128,0,1,float16,float16,0,0.9074347019195557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,24,4,128,0,1,float16,float16,0,0.49032533168792725
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,24,8,128,0,1,float16,float16,0,0.49698134263356525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,24,2,128,0,1,float16,float16,0,0.4927146832148234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,24,24,128,0,1,float16,float16,0,0.5077333450317383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,24,1,128,0,1,float16,float16,0,0.28859732548395794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,24,24,128,0,1,float16,float16,0,0.29713066418965656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,24,2,128,0,1,float16,float16,0,0.2887679934501648
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,24,4,128,0,1,float16,float16,0,0.29064534107844037
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,24,8,128,0,1,float16,float16,0,0.2949120004971822
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,24,1,128,0,1,float16,float16,0,4.118869463602702
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,24,2,128,0,1,float16,float16,0,4.122453371683757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,24,4,128,0,1,float16,float16,0,4.139349301656087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1536,24,1,128,0,1,float16,float16,0,8.4759890238444
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,24,1,128,0,1,float16,float16,0,2.0374186833699546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1536,24,2,128,0,1,float16,float16,0,8.518997192382812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1536,24,4,128,0,1,float16,float16,0,8.587776184082031
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,24,24,128,0,1,float16,float16,0,4.5182294845581055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,24,2,128,0,1,float16,float16,0,2.051754633585612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,24,4,128,0,1,float16,float16,0,2.0587520599365234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,24,8,128,0,1,float16,float16,0,4.231338818868001
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,24,24,128,0,1,float16,float16,0,2.241706689198812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,24,1,128,0,1,float16,float16,0,1.0482347011566162
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,24,2,128,0,1,float16,float16,0,1.0525013605753581
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,24,4,128,0,1,float16,float16,0,1.055402676264445
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,24,24,128,0,1,float16,float16,0,1.1477333704630535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,24,8,128,0,1,float16,float16,0,2.110293388366699
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,24,24,128,0,1,float16,float16,0,0.5975040197372437
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,24,8,128,0,1,float16,float16,0,1.0693973700205486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,24,1,128,0,1,float16,float16,0,0.556714653968811
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,24,2,128,0,1,float16,float16,0,0.5609813531239828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1536,24,8,128,0,1,float16,float16,0,8.706560134887695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,24,4,128,0,1,float16,float16,0,0.5597866773605347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,24,1,128,0,1,float16,float16,0,0.3109546701113383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,24,2,128,0,1,float16,float16,0,0.3109546701113383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,24,4,128,0,1,float16,float16,0,0.3136853377024333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,24,8,128,0,1,float16,float16,0,0.5657600164413452
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,24,8,128,0,1,float16,float16,0,0.3155626654624939
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,24,24,128,0,1,float16,float16,0,0.3298986752827962
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,24,2,128,0,1,float16,float16,0,0.19114667177200317
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,24,1,128,0,1,float16,float16,0,0.18824533621470133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,24,4,128,0,1,float16,float16,0,0.18978132804234824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,24,8,128,0,1,float16,float16,0,0.1914880077044169
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,24,24,128,0,1,float16,float16,0,0.19387733936309814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,24,1,128,0,1,float16,float16,0,4.010666529337565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,24,2,128,0,1,float16,float16,0,4.043946584065755
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,24,4,128,0,1,float16,float16,0,4.142079989115397
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,1024,24,1,128,0,1,float16,float16,0,8.227669397989908
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,1024,24,2,128,0,1,float16,float16,0,8.355840047200521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,1024,24,4,128,0,1,float16,float16,0,8.410453160603842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,24,1,128,0,1,float16,float16,0,2.0077226956685386
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,24,2,128,0,1,float16,float16,0,2.0365653038024902
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,24,24,128,0,1,float16,float16,0,4.764330546061198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,24,24,128,0,1,float16,float16,0,2.3610026041666665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,24,4,128,0,1,float16,float16,0,2.0638720194498696
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,24,8,128,0,1,float16,float16,0,4.203349431355794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,24,1,128,0,1,float16,float16,0,1.0263893604278564
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,24,2,128,0,1,float16,float16,0,1.0367999871571858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,1024,24,8,128,0,1,float16,float16,0,8.502101262410482
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,24,4,128,0,1,float16,float16,0,1.047210693359375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,24,1,128,0,1,float16,float16,0,0.5348693529764811
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,24,8,128,0,1,float16,float16,0,2.1084159215291343
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,24,2,128,0,1,float16,float16,0,0.5369173288345337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,24,24,128,0,1,float16,float16,0,1.1721386909484863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,24,8,128,0,1,float16,float16,0,1.061205307642619
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,24,24,128,0,1,float16,float16,0,0.609279990196228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,24,4,128,0,1,float16,float16,0,0.5451093514760336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,24,8,128,0,1,float16,float16,0,0.549888014793396
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,24,1,128,0,1,float16,float16,0,0.28808534145355225
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,24,2,128,0,1,float16,float16,0,0.28996266921361286
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,24,8,128,0,1,float16,float16,0,0.2979840040206909
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,24,24,128,0,1,float16,float16,0,0.3217066725095113
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,24,1,128,0,1,float16,float16,0,0.159061332543691
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,24,24,128,0,1,float16,float16,0,0.18107734123865762
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,24,4,128,0,1,float16,float16,0,0.29371732473373413
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,24,2,128,0,1,float16,float16,0,0.16110933820406595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,24,4,128,0,1,float16,float16,0,0.16264533003171286
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,24,8,128,0,1,float16,float16,0,0.16503467162450156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,24,1,128,0,1,float16,float16,0,0.09949866930643718
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,24,24,128,0,1,float16,float16,0,0.10598400235176086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,24,4,128,0,1,float16,float16,0,0.10103467106819153
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,24,2,128,0,1,float16,float16,0,0.0993280013402303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,24,8,128,0,1,float16,float16,0,0.10154666503270467
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,24,1,128,0,1,float16,float16,0,2.719231923421224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,24,2,128,0,1,float16,float16,0,2.775381406148275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,24,4,128,0,1,float16,float16,0,2.838186581929525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,512,24,1,128,0,1,float16,float16,0,5.478570938110352
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,512,24,2,128,0,1,float16,float16,0,5.625002543131511
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,24,1,128,0,1,float16,float16,0,1.3685760498046875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,512,24,4,128,0,1,float16,float16,0,5.779285430908203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,24,2,128,0,1,float16,float16,0,1.394858678181966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,24,8,128,0,1,float16,float16,0,2.9257386525472007
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,24,24,128,0,1,float16,float16,0,1.7240746815999348
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,24,4,128,0,1,float16,float16,0,1.4272853533426921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,24,1,128,0,1,float16,float16,0,0.7004160086313883
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,24,24,128,0,1,float16,float16,0,3.4435412089029946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,24,2,128,0,1,float16,float16,0,0.7082666556040446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,512,24,8,128,0,1,float16,float16,0,6.039893468221028
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,24,4,128,0,1,float16,float16,0,0.71833602587382
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,24,1,128,0,1,float16,float16,0,0.3657386700312297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,24,24,128,0,1,float16,float16,0,0.868010679880778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,24,2,128,0,1,float16,float16,0,0.36744534969329834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,24,4,128,0,1,float16,float16,0,0.374783992767334
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,24,24,128,0,1,float16,float16,0,0.44014934698740643
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,24,8,128,0,1,float16,float16,0,0.7406933307647705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,24,8,128,0,1,float16,float16,0,1.4574933052062988
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,24,8,128,0,1,float16,float16,0,0.38075733184814453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,24,1,128,0,1,float16,float16,0,0.1967786749204
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,24,24,128,0,1,float16,float16,0,0.2315946618715922
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,24,2,128,0,1,float16,float16,0,0.19950934251149496
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,24,4,128,0,1,float16,float16,0,0.20087466637293497
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,24,8,128,0,1,float16,float16,0,0.20497065782546997
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,24,24,128,0,1,float16,float16,0,0.13056000073750815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,24,1,128,0,1,float16,float16,0,0.10769066214561462
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,24,4,128,0,1,float16,float16,0,0.11161599556605022
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,24,24,128,0,1,float16,float16,0,0.07423999905586243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,24,8,128,0,1,float16,float16,0,0.1155413289864858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,24,1,128,0,1,float16,float16,0,0.067071999112765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,24,2,128,0,1,float16,float16,0,0.10990933577219646
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,24,2,128,0,1,float16,float16,0,0.06929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,24,4,128,0,1,float16,float16,0,0.06809600194295247
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,24,24,128,0,1,float16,float16,0,0.04147200038035711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,24,1,128,0,1,float16,float16,0,0.039594667653242745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,24,2,128,0,1,float16,float16,0,0.03942399968703588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,24,8,128,0,1,float16,float16,0,0.06929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,24,4,128,0,1,float16,float16,0,0.039936001102129616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,24,8,128,0,1,float16,float16,0,0.040106666584809623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,24,1,128,0,1,float16,float16,0,1.0499413013458252
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,24,2,128,0,1,float16,float16,0,1.0745173295338948
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,24,4,128,0,1,float16,float16,0,1.098410685857137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,256,24,1,128,0,1,float16,float16,0,2.090837319691976
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,256,24,2,128,0,1,float16,float16,0,2.1597867012023926
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,24,1,128,0,1,float16,float16,0,0.5403306484222412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,256,24,4,128,0,1,float16,float16,0,2.2181545893351235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,24,2,128,0,1,float16,float16,0,0.5492053429285685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,24,8,128,0,1,float16,float16,0,1.1422719955444336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,24,24,128,0,1,float16,float16,0,0.7232853571573893
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,24,24,128,0,1,float16,float16,0,1.423701286315918
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,256,24,8,128,0,1,float16,float16,0,2.312191963195801
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,24,4,128,0,1,float16,float16,0,0.5584213336308798
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,24,1,128,0,1,float16,float16,0,0.2797226707140605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,24,24,128,0,1,float16,float16,0,0.3797333240509033
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,24,8,128,0,1,float16,float16,0,0.5771946509679159
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,24,2,128,0,1,float16,float16,0,0.285866657892863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,24,1,128,0,1,float16,float16,0,0.15018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,24,2,128,0,1,float16,float16,0,0.1527466674645742
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,24,4,128,0,1,float16,float16,0,0.2903040051460266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,24,24,128,0,1,float16,float16,0,0.20172800620396933
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,24,8,128,0,1,float16,float16,0,0.30003199974695843
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,24,4,128,0,1,float16,float16,0,0.155648003021876
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,24,1,128,0,1,float16,float16,0,0.0817493349313736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,24,2,128,0,1,float16,float16,0,0.08413867155710857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,24,8,128,0,1,float16,float16,0,0.1609386702378591
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,24,4,128,0,1,float16,float16,0,0.08482133348782857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,24,24,128,0,1,float16,float16,0,0.056832000613212585
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,24,8,128,0,1,float16,float16,0,0.08925867080688477
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,24,1,128,0,1,float16,float16,0,0.04983466863632202
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,24,24,128,0,1,float16,float16,0,0.11025066177050273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,24,2,128,0,1,float16,float16,0,0.05085866649945577
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,24,4,128,0,1,float16,float16,0,0.05085866649945577
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,24,8,128,0,1,float16,float16,0,0.051882664362589516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,24,24,128,0,1,float16,float16,0,0.03328000009059906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,24,2,128,0,1,float16,float16,0,0.0314026673634847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,24,1,128,0,1,float16,float16,0,0.0314026673634847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,24,4,128,0,1,float16,float16,0,0.031914666295051575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,24,1,128,0,1,float16,float16,0,0.024234667420387268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,24,24,128,0,1,float16,float16,0,0.02491733431816101
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,24,2,128,0,1,float16,float16,0,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,24,4,128,0,1,float16,float16,0,0.024405332903067272
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,24,8,128,0,1,float16,float16,0,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,24,8,128,0,1,float16,float16,0,0.031914666295051575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,24,1,128,0,1,float16,float16,0,0.46062934398651123
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,24,2,128,0,1,float16,float16,0,0.47223468621571857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,128,24,1,128,0,1,float16,float16,0,0.9055573145548502
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,128,24,2,128,0,1,float16,float16,0,0.9303039709726969
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,128,24,4,128,0,1,float16,float16,0,0.9530026912689209
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,24,4,128,0,1,float16,float16,0,0.48349865277608234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,24,1,128,0,1,float16,float16,0,0.2402986685434977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,24,2,128,0,1,float16,float16,0,0.24780799945195517
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,24,24,128,0,1,float16,float16,0,0.6876160303751627
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,128,24,8,128,0,1,float16,float16,0,1.0018133322397869
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,24,8,128,0,1,float16,float16,0,0.5031253496805826
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,24,24,128,0,1,float16,float16,0,0.3520853519439697
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,24,4,128,0,1,float16,float16,0,0.2515626748402913
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,24,1,128,0,1,float16,float16,0,0.1281706690788269
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,24,2,128,0,1,float16,float16,0,0.13363200426101685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,24,4,128,0,1,float16,float16,0,0.1365333298842112
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,24,8,128,0,1,float16,float16,0,0.2606079975763957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,24,24,128,0,1,float16,float16,0,0.187391996383667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,24,1,128,0,1,float16,float16,0,0.06843733290831248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,24,8,128,0,1,float16,float16,0,0.14250666896502176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,24,24,128,0,1,float16,float16,0,0.10342400272687276
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,24,2,128,0,1,float16,float16,0,0.07014399766921997
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,24,4,128,0,1,float16,float16,0,0.07253333429495494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,24,1,128,0,1,float16,float16,0,0.039936001102129616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,24,8,128,0,1,float16,float16,0,0.0769706666469574
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,24,24,128,0,1,float16,float16,0,0.04744533201058706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,24,2,128,0,1,float16,float16,0,0.040448000033696495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,24,4,128,0,1,float16,float16,0,0.04113066693147024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,24,8,128,0,1,float16,float16,0,0.04130133241415024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,24,24,128,0,1,float16,float16,0,0.027136000494162243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,24,1,128,0,1,float16,float16,0,0.025770666698614757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,24,2,128,0,1,float16,float16,0,0.025770666698614757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,24,4,128,0,1,float16,float16,0,0.02611200014750163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,24,1,128,0,1,float16,float16,0,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,24,24,128,0,1,float16,float16,0,0.020309332758188248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,24,8,128,0,1,float16,float16,0,0.02628266563018163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,24,4,128,0,1,float16,float16,0,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,24,8,128,0,1,float16,float16,0,0.019968000551064808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,24,24,128,0,1,float16,float16,0,0.01757866640885671
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,24,1,128,0,1,float16,float16,0,0.01757866640885671
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,24,4,128,0,1,float16,float16,0,0.01757866640885671
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,24,2,128,0,1,float16,float16,0,0.017407999684413273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,24,8,128,0,1,float16,float16,0,0.01757866640885671
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,24,2,128,0,1,float16,float16,0,0.024405332903067272
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,64,24,1,128,0,1,float16,float16,0,0.4493653376897176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,24,1,128,0,1,float16,float16,0,0.23893332481384277
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,64,24,2,128,0,1,float16,float16,0,0.46199464797973633
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,24,2,128,0,1,float16,float16,0,0.24473599592844644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,64,24,4,128,0,1,float16,float16,0,0.48213334878285724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,24,4,128,0,1,float16,float16,0,0.2524159948031108
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,24,24,128,0,1,float16,float16,0,0.3563520113627116
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,24,1,128,0,1,float16,float16,0,0.1276586651802063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,64,24,8,128,0,1,float16,float16,0,0.5090986490249634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,24,2,128,0,1,float16,float16,0,0.13090133666992188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,24,4,128,0,1,float16,float16,0,0.1397760013739268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,24,8,128,0,1,float16,float16,0,0.265557328859965
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,24,24,128,0,1,float16,float16,0,0.18926932414372763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,24,8,128,0,1,float16,float16,0,0.14421332875887552
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,24,1,128,0,1,float16,float16,0,0.06690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,24,24,128,0,1,float16,float16,0,0.10769066214561462
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,24,2,128,0,1,float16,float16,0,0.06877866884072621
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,24,4,128,0,1,float16,float16,0,0.07099733253320058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,24,1,128,0,1,float16,float16,0,0.03976533313592275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,24,24,128,0,1,float16,float16,0,0.04744533201058706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,24,2,128,0,1,float16,float16,0,0.040106666584809623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,24,8,128,0,1,float16,float16,0,0.07765333354473114
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,24,4,128,0,1,float16,float16,0,0.040448000033696495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,24,8,128,0,1,float16,float16,0,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,24,2,128,0,1,float16,float16,0,0.025258667767047882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,24,24,128,0,1,float16,float16,0,0.0266239990790685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,24,1,128,0,1,float16,float16,0,0.02491733431816101
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,24,4,128,0,1,float16,float16,0,0.025770666698614757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,24,8,128,0,1,float16,float16,0,0.025770666698614757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,24,24,128,0,1,float16,float16,0,0.019968000551064808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,24,1,128,0,1,float16,float16,0,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,24,2,128,0,1,float16,float16,0,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,24,4,128,0,1,float16,float16,0,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,24,8,128,0,1,float16,float16,0,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,24,1,128,0,1,float16,float16,0,0.017407999684413273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,24,2,128,0,1,float16,float16,0,0.017237332959969837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,24,24,128,0,1,float16,float16,0,0.016895999511082966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,24,4,128,0,1,float16,float16,0,0.017407999684413273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,24,8,128,0,1,float16,float16,0,0.017407999684413273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,24,24,128,0,1,float16,float16,0,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,24,1,128,0,1,float16,float16,0,0.016554666062196095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,24,2,128,0,1,float16,float16,0,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,24,8,128,0,1,float16,float16,0,0.01672533278663953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,24,4,128,0,1,float16,float16,0,0.016554666062196095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,32,24,1,128,0,1,float16,float16,0,0.26368000109990436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,32,24,2,128,0,1,float16,float16,0,0.2677759925524394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,24,2,128,0,1,float16,float16,0,0.14335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,24,1,128,0,1,float16,float16,0,0.13960533340771994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,32,24,4,128,0,1,float16,float16,0,0.27135999997456867
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,24,4,128,0,1,float16,float16,0,0.14523733655611673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,32,24,8,128,0,1,float16,float16,0,0.30054400364557904
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,24,24,128,0,1,float16,float16,0,0.1919999917348226
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,24,24,128,0,1,float16,float16,0,0.10496000448862712
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,24,8,128,0,1,float16,float16,0,0.15155200163523355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,24,2,128,0,1,float16,float16,0,0.07577600081761678
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,24,4,128,0,1,float16,float16,0,0.07850666840871175
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,24,8,128,0,1,float16,float16,0,0.0817493349313736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,24,24,128,0,1,float16,float16,0,0.048469334840774536
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,24,1,128,0,1,float16,float16,0,0.041984001795450844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,24,1,128,0,1,float16,float16,0,0.07355733215808868
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,24,2,128,0,1,float16,float16,0,0.04232533276081085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,24,4,128,0,1,float16,float16,0,0.042837331692377724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,24,24,128,0,1,float16,float16,0,0.027477333943049114
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,24,8,128,0,1,float16,float16,0,0.04351999859015147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,24,1,128,0,1,float16,float16,0,0.02679466704527537
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,24,2,128,0,1,float16,float16,0,0.026965332527955372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,24,4,128,0,1,float16,float16,0,0.027136000494162243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,24,8,128,0,1,float16,float16,0,0.027477333943049114
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,24,24,128,0,1,float16,float16,0,0.018602666755517323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,24,2,128,0,1,float16,float16,0,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,24,4,128,0,1,float16,float16,0,0.018602666755517323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,24,1,128,0,1,float16,float16,0,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,24,1,128,0,1,float16,float16,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,24,24,128,0,1,float16,float16,0,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,24,8,128,0,1,float16,float16,0,0.018602666755517323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,24,2,128,0,1,float16,float16,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,24,4,128,0,1,float16,float16,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,24,8,128,0,1,float16,float16,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,24,24,128,0,1,float16,float16,0,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,24,1,128,0,1,float16,float16,0,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,24,2,128,0,1,float16,float16,0,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,24,4,128,0,1,float16,float16,0,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,24,8,128,0,1,float16,float16,0,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,24,24,128,0,1,float16,float16,0,0.013823999712864557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,24,1,128,0,1,float16,float16,0,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,24,2,128,0,1,float16,float16,0,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,24,4,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,24,8,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,16,24,2,128,0,1,float16,float16,0,0.20838399728139242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,16,24,4,128,0,1,float16,float16,0,0.21145600080490112
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,16,24,1,128,0,1,float16,float16,0,0.2058239976565043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,16,24,8,128,0,1,float16,float16,0,0.2182826598485311
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,24,1,128,0,1,float16,float16,0,0.10478933652242024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,24,24,128,0,1,float16,float16,0,0.1250986655553182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,24,4,128,0,1,float16,float16,0,0.10922666390736897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,24,2,128,0,1,float16,float16,0,0.10734933614730835
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,24,1,128,0,1,float16,float16,0,0.0580266664425532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,24,2,128,0,1,float16,float16,0,0.05870933334032694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,24,8,128,0,1,float16,float16,0,0.11451733112335205
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,24,24,128,0,1,float16,float16,0,0.0628053347269694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,24,8,128,0,1,float16,float16,0,0.05973333120346069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,24,4,128,0,1,float16,float16,0,0.059392000238100685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,24,1,128,0,1,float16,float16,0,0.03515733281771342
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,24,2,128,0,1,float16,float16,0,0.03515733281771342
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,24,4,128,0,1,float16,float16,0,0.03566933423280716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,24,8,128,0,1,float16,float16,0,0.03549866626660029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,24,1,128,0,1,float16,float16,0,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,24,24,128,0,1,float16,float16,0,0.034815999368826546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,24,2,128,0,1,float16,float16,0,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,24,24,128,0,1,float16,float16,0,0.022357332209746044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,24,4,128,0,1,float16,float16,0,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,24,8,128,0,1,float16,float16,0,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,24,24,128,0,1,float16,float16,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,24,1,128,0,1,float16,float16,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,24,2,128,0,1,float16,float16,0,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,24,24,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,24,8,128,0,1,float16,float16,0,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,24,1,128,0,1,float16,float16,0,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,24,2,128,0,1,float16,float16,0,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,24,4,128,0,1,float16,float16,0,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,24,24,128,0,1,float16,float16,0,0.013823999712864557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,24,8,128,0,1,float16,float16,0,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,24,1,128,0,1,float16,float16,0,0.013823999712864557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,24,2,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,24,4,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,24,4,128,0,1,float16,float16,0,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,24,8,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,24,24,128,0,1,float16,float16,0,0.013653332988421122
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,24,1,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,24,2,128,0,1,float16,float16,0,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,24,4,128,0,1,float16,float16,0,0.013653332988421122
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,24,8,128,0,1,float16,float16,0,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,16,1,128,0,1,float16,float16,0,29.388287862141926
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,16,2,128,0,1,float16,float16,0,29.81495412190755
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,16,4,128,0,1,float16,float16,0,29.112319946289062
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16384,16,1,128,0,1,float16,float16,0,58.95116678873698
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,16,1,128,0,1,float16,float16,0,14.681770324707031
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,16,16,128,0,1,float16,float16,0,30.130859375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,16,16,128,0,1,float16,float16,0,15.209643046061197
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16384,16,2,128,0,1,float16,float16,0,59.53996785481771
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16384,16,4,128,0,1,float16,float16,0,59.90416971842448
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,16,8,128,0,1,float16,float16,0,29.472938537597656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,16,2,128,0,1,float16,float16,0,15.025151570638021
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,16,1,128,0,1,float16,float16,0,7.7269331614176435
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,16,4,128,0,1,float16,float16,0,14.649344126383463
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,16,16,128,0,1,float16,float16,0,7.785301208496094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,16,2,128,0,1,float16,float16,0,7.768063863118489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,16,8,128,0,1,float16,float16,0,15.202475229899088
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16384,16,8,128,0,1,float16,float16,0,58.61768595377604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,16,4,128,0,1,float16,float16,0,7.595008214314778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,16,8,128,0,1,float16,float16,0,7.732224146525065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,16,1,128,0,1,float16,float16,0,16.8448003133138
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,16,2,128,0,1,float16,float16,0,16.835243225097656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,16,4,128,0,1,float16,float16,0,17.016490936279297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,12288,16,1,128,0,1,float16,float16,0,33.98877970377604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,12288,16,2,128,0,1,float16,float16,0,33.0238291422526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,16,16,128,0,1,float16,float16,0,17.167701721191406
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,16,1,128,0,1,float16,float16,0,8.587434768676758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,12288,16,4,128,0,1,float16,float16,0,33.79199981689453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,16,16,128,0,1,float16,float16,0,8.599552154541016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,16,2,128,0,1,float16,float16,0,8.626858393351236
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,16,4,128,0,1,float16,float16,0,8.392362594604492
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,16,8,128,0,1,float16,float16,0,16.874837239583332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,16,1,128,0,1,float16,float16,0,4.407807985941569
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,16,2,128,0,1,float16,float16,0,4.3852799733479815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,12288,16,8,128,0,1,float16,float16,0,33.83108266194662
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,16,16,128,0,1,float16,float16,0,4.428970654805501
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,16,8,128,0,1,float16,float16,0,8.657920201619467
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,16,8,128,0,1,float16,float16,0,4.3692372639973955
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,16,4,128,0,1,float16,float16,0,4.386133193969727
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,16,1,128,0,1,float16,float16,0,11.884031931559244
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,16,2,128,0,1,float16,float16,0,11.627690633138021
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,10240,16,1,128,0,1,float16,float16,0,23.427584330240887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,10240,16,2,128,0,1,float16,float16,0,23.79298146565755
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,16,4,128,0,1,float16,float16,0,12.028586069742838
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,10240,16,4,128,0,1,float16,float16,0,23.489023844401043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,16,16,128,0,1,float16,float16,0,11.993258158365885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,16,1,128,0,1,float16,float16,0,5.988181432088216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,16,2,128,0,1,float16,float16,0,6.111743927001953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,16,16,128,0,1,float16,float16,0,6.004053115844727
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,16,8,128,0,1,float16,float16,0,11.997013092041016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,16,4,128,0,1,float16,float16,0,6.0868269602457685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,16,1,128,0,1,float16,float16,0,3.138218561808268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,10240,16,8,128,0,1,float16,float16,0,23.80578104654948
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,16,2,128,0,1,float16,float16,0,3.138901392618815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,16,8,128,0,1,float16,float16,0,5.988522847493489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,16,16,128,0,1,float16,float16,0,3.168426513671875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,16,4,128,0,1,float16,float16,0,3.126784006754557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,16,8,128,0,1,float16,float16,0,3.1320746739705405
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,16,1,128,0,1,float16,float16,0,15.310335795084635
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,16,2,128,0,1,float16,float16,0,15.14786148071289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,16,4,128,0,1,float16,float16,0,15.465471903483072
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,8192,16,1,128,0,1,float16,float16,0,30.623062133789062
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,8192,16,2,128,0,1,float16,float16,0,30.401535034179688
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,16,1,128,0,1,float16,float16,0,7.723520278930664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,16,16,128,0,1,float16,float16,0,15.4519894917806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,8192,16,4,128,0,1,float16,float16,0,30.73894500732422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,16,16,128,0,1,float16,float16,0,7.6931413014729815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,16,4,128,0,1,float16,float16,0,7.6909224192301435
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,16,2,128,0,1,float16,float16,0,7.809024175008138
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,16,1,128,0,1,float16,float16,0,3.917824109395345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,16,8,128,0,1,float16,float16,0,15.379114786783854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,16,16,128,0,1,float16,float16,0,3.9468374252319336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,16,2,128,0,1,float16,float16,0,3.8737920125325522
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,8192,16,8,128,0,1,float16,float16,0,31.18165334065755
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,16,8,128,0,1,float16,float16,0,7.802879969278972
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,16,4,128,0,1,float16,float16,0,3.8867626190185547
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,16,1,128,0,1,float16,float16,0,2.0862293243408203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,16,2,128,0,1,float16,float16,0,2.0758186976114907
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,16,16,128,0,1,float16,float16,0,2.097493330637614
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,16,4,128,0,1,float16,float16,0,2.079744021097819
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,16,8,128,0,1,float16,float16,0,2.100053310394287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,16,8,128,0,1,float16,float16,0,3.9509334564208984
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,16,1,128,0,1,float16,float16,0,8.667306900024414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,16,2,128,0,1,float16,float16,0,8.962048212687174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,16,4,128,0,1,float16,float16,0,8.98969586690267
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,6144,16,1,128,0,1,float16,float16,0,17.72492853800456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,6144,16,2,128,0,1,float16,float16,0,17.58566411336263
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,16,1,128,0,1,float16,float16,0,4.427263895670573
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,16,16,128,0,1,float16,float16,0,9.105578740437826
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,6144,16,4,128,0,1,float16,float16,0,17.645909627278645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,16,2,128,0,1,float16,float16,0,4.4397226969401045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,16,16,128,0,1,float16,float16,0,4.561749458312988
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,16,4,128,0,1,float16,float16,0,4.418389320373535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,16,8,128,0,1,float16,float16,0,8.727551778157553
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,16,1,128,0,1,float16,float16,0,2.2973440488179526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,16,2,128,0,1,float16,float16,0,2.2853973706563315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,6144,16,8,128,0,1,float16,float16,0,17.62508773803711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,16,16,128,0,1,float16,float16,0,2.321066697438558
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,16,4,128,0,1,float16,float16,0,2.291541258494059
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,16,1,128,0,1,float16,float16,0,1.2632746696472168
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,16,2,128,0,1,float16,float16,0,1.2588373025258381
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,16,8,128,0,1,float16,float16,0,2.285226662953695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,16,4,128,0,1,float16,float16,0,1.268053372701009
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,16,16,128,0,1,float16,float16,0,1.2602026462554932
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,16,8,128,0,1,float16,float16,0,4.496554692586263
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,16,8,128,0,1,float16,float16,0,1.2545706431070964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,16,1,128,0,1,float16,float16,0,8.046592076619467
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,16,2,128,0,1,float16,float16,0,8.33177630106608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,16,4,128,0,1,float16,float16,0,8.260437647501627
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,4096,16,1,128,0,1,float16,float16,0,16.47650146484375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,4096,16,2,128,0,1,float16,float16,0,16.526507059733074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,16,1,128,0,1,float16,float16,0,4.069375991821289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,16,16,128,0,1,float16,float16,0,8.402432123819986
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,4096,16,4,128,0,1,float16,float16,0,16.69051742553711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,16,2,128,0,1,float16,float16,0,4.101973215738933
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,16,4,128,0,1,float16,float16,0,4.095487912495931
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,16,1,128,0,1,float16,float16,0,2.074965318044027
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,16,16,128,0,1,float16,float16,0,4.212053298950195
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,16,8,128,0,1,float16,float16,0,8.153770446777344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,16,2,128,0,1,float16,float16,0,2.080085277557373
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,16,16,128,0,1,float16,float16,0,2.1353813807169595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,16,1,128,0,1,float16,float16,0,1.1106986999511719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,4096,16,8,128,0,1,float16,float16,0,16.571392059326172
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,16,2,128,0,1,float16,float16,0,1.1084799766540527
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,16,8,128,0,1,float16,float16,0,4.117845217386882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,16,4,128,0,1,float16,float16,0,2.0879359245300293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,16,16,128,0,1,float16,float16,0,1.129642645517985
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,16,4,128,0,1,float16,float16,0,1.1113813718159993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,16,8,128,0,1,float16,float16,0,2.093056042989095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,16,1,128,0,1,float16,float16,0,0.6248106559117635
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,16,16,128,0,1,float16,float16,0,0.6326613426208496
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,16,2,128,0,1,float16,float16,0,0.6283946832021078
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,16,8,128,0,1,float16,float16,0,1.1194026470184326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,16,4,128,0,1,float16,float16,0,0.6350506544113159
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,16,8,128,0,1,float16,float16,0,0.6331733465194702
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,16,1,128,0,1,float16,float16,0,4.723370552062988
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,16,2,128,0,1,float16,float16,0,4.830719947814941
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,16,4,128,0,1,float16,float16,0,4.874581336975098
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,3072,16,1,128,0,1,float16,float16,0,9.475754419962565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,3072,16,2,128,0,1,float16,float16,0,9.725610733032227
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,3072,16,4,128,0,1,float16,float16,0,9.72543970743815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,16,1,128,0,1,float16,float16,0,2.3659520149230957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,16,2,128,0,1,float16,float16,0,2.379605293273926
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,16,16,128,0,1,float16,float16,0,5.075455983479817
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,16,16,128,0,1,float16,float16,0,2.501973311106364
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,16,4,128,0,1,float16,float16,0,2.3900160789489746
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,16,1,128,0,1,float16,float16,0,1.2339200178782146
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,16,8,128,0,1,float16,float16,0,4.791808128356934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,16,2,128,0,1,float16,float16,0,1.2484266757965088
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,16,4,128,0,1,float16,float16,0,1.2533760070800781
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,3072,16,8,128,0,1,float16,float16,0,9.749162673950195
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,16,1,128,0,1,float16,float16,0,0.66594131787618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,16,2,128,0,1,float16,float16,0,0.6758399804433187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,16,8,128,0,1,float16,float16,0,2.411349296569824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,16,16,128,0,1,float16,float16,0,1.2823893229166667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,16,8,128,0,1,float16,float16,0,1.256106694539388
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,16,16,128,0,1,float16,float16,0,0.6886400381724039
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,16,4,128,0,1,float16,float16,0,0.6789120038350424
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,16,1,128,0,1,float16,float16,0,0.3952639897664388
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,16,2,128,0,1,float16,float16,0,0.3969706694285075
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,16,16,128,0,1,float16,float16,0,0.39953064918518066
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,16,8,128,0,1,float16,float16,0,0.6830080350240072
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,16,4,128,0,1,float16,float16,0,0.39816534519195557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,16,8,128,0,1,float16,float16,0,0.40089599291483563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,16,1,128,0,1,float16,float16,0,4.569087982177734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,16,2,128,0,1,float16,float16,0,4.648960113525391
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,16,4,128,0,1,float16,float16,0,4.691455841064453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,2048,16,1,128,0,1,float16,float16,0,9.222485224405924
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,16,1,128,0,1,float16,float16,0,2.272256056467692
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,2048,16,2,128,0,1,float16,float16,0,9.388885498046875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,16,2,128,0,1,float16,float16,0,2.2860800425211587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,16,16,128,0,1,float16,float16,0,4.846933364868164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,2048,16,4,128,0,1,float16,float16,0,9.440938949584961
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,16,16,128,0,1,float16,float16,0,2.423807938893636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,16,4,128,0,1,float16,float16,0,2.304170608520508
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,16,8,128,0,1,float16,float16,0,4.709888140360515
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,16,1,128,0,1,float16,float16,0,1.160533348719279
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,16,2,128,0,1,float16,float16,0,1.1717973550160725
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,2048,16,8,128,0,1,float16,float16,0,9.596245447794596
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,16,16,128,0,1,float16,float16,0,1.2381866772969563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,16,1,128,0,1,float16,float16,0,0.6178133487701416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,16,2,128,0,1,float16,float16,0,0.617301344871521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,16,16,128,0,1,float16,float16,0,0.6487040122350057
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,16,4,128,0,1,float16,float16,0,1.1774293581644695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,16,8,128,0,1,float16,float16,0,2.357247988382975
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,16,8,128,0,1,float16,float16,0,1.1898880004882812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,16,4,128,0,1,float16,float16,0,0.6272000074386597
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,16,16,128,0,1,float16,float16,0,0.3604480028152466
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,16,8,128,0,1,float16,float16,0,0.6311253309249878
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,16,1,128,0,1,float16,float16,0,0.34594134489695233
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,16,2,128,0,1,float16,float16,0,0.3495253324508667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,16,16,128,0,1,float16,float16,0,0.23688532908757529
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,16,1,128,0,1,float16,float16,0,0.21145600080490112
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,16,8,128,0,1,float16,float16,0,0.353279987970988
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,16,2,128,0,1,float16,float16,0,0.21094399690628052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,16,4,128,0,1,float16,float16,0,0.34645334879557294
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,16,4,128,0,1,float16,float16,0,0.21248000860214233
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,16,8,128,0,1,float16,float16,0,0.21504000822703043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,16,2,128,0,1,float16,float16,0,2.7779413859049478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,16,1,128,0,1,float16,float16,0,2.7641172409057617
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,16,4,128,0,1,float16,float16,0,2.8117332458496094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1536,16,1,128,0,1,float16,float16,0,5.555370966593425
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1536,16,2,128,0,1,float16,float16,0,5.637290954589844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,16,1,128,0,1,float16,float16,0,1.384618600209554
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1536,16,4,128,0,1,float16,float16,0,5.736277262369792
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,16,2,128,0,1,float16,float16,0,1.3982720375061035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,16,16,128,0,1,float16,float16,0,3.003562609354655
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,16,16,128,0,1,float16,float16,0,1.504426638285319
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,16,4,128,0,1,float16,float16,0,1.4103892644246419
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,16,1,128,0,1,float16,float16,0,0.7167999744415283
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,16,8,128,0,1,float16,float16,0,2.8583253224690757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,16,2,128,0,1,float16,float16,0,0.7255040009816488
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,16,16,128,0,1,float16,float16,0,0.7780693372090658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,16,8,128,0,1,float16,float16,0,1.4375252723693848
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,16,1,128,0,1,float16,float16,0,0.38997332255045575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1536,16,8,128,0,1,float16,float16,0,5.8301442464192705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,16,2,128,0,1,float16,float16,0,0.38860801855723065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,16,4,128,0,1,float16,float16,0,0.7253333727518717
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,16,16,128,0,1,float16,float16,0,0.4116479953130086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,16,8,128,0,1,float16,float16,0,0.7377920150756836
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,16,4,128,0,1,float16,float16,0,0.3959466616312663
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,16,8,128,0,1,float16,float16,0,0.39662933349609375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,16,2,128,0,1,float16,float16,0,0.2254506746927897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,16,1,128,0,1,float16,float16,0,0.222378671169281
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,16,16,128,0,1,float16,float16,0,0.2336426575978597
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,16,1,128,0,1,float16,float16,0,0.14574933052062988
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,16,16,128,0,1,float16,float16,0,0.1469439963499705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,16,8,128,0,1,float16,float16,0,0.22425599892934164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,16,2,128,0,1,float16,float16,0,0.14506666858990988
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,16,4,128,0,1,float16,float16,0,0.22613332668940225
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,16,4,128,0,1,float16,float16,0,0.14353066682815552
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,16,8,128,0,1,float16,float16,0,0.14660267035166422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,16,2,128,0,1,float16,float16,0,2.7267414728800454
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,16,4,128,0,1,float16,float16,0,2.7779413859049478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,16,1,128,0,1,float16,float16,0,2.6774187088012695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,1024,16,1,128,0,1,float16,float16,0,5.381631851196289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,1024,16,2,128,0,1,float16,float16,0,5.535743713378906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,16,1,128,0,1,float16,float16,0,1.3579947153727214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,16,2,128,0,1,float16,float16,0,1.377962589263916
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,1024,16,4,128,0,1,float16,float16,0,5.64241091410319
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,16,16,128,0,1,float16,float16,0,3.1276372273763022
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,16,4,128,0,1,float16,float16,0,1.3963947296142578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,16,16,128,0,1,float16,float16,0,1.5709865887959797
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,16,1,128,0,1,float16,float16,0,0.7108266353607178
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,16,2,128,0,1,float16,float16,0,0.7099733352661133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,16,8,128,0,1,float16,float16,0,2.8477439880371094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,16,16,128,0,1,float16,float16,0,0.794111967086792
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,1024,16,8,128,0,1,float16,float16,0,5.783722559611003
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,16,4,128,0,1,float16,float16,0,0.716970682144165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,16,1,128,0,1,float16,float16,0,0.3684693177541097
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,16,16,128,0,1,float16,float16,0,0.4247893492380778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,16,2,128,0,1,float16,float16,0,0.3974826733271281
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,16,8,128,0,1,float16,float16,0,0.7372799714406332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,16,1,128,0,1,float16,float16,0,0.2034346659978231
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,16,4,128,0,1,float16,float16,0,0.37887998421986896
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,16,16,128,0,1,float16,float16,0,0.2254506746927897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,16,8,128,0,1,float16,float16,0,0.38690133889516193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,16,8,128,0,1,float16,float16,0,1.4341120719909668
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,16,2,128,0,1,float16,float16,0,0.2034346659978231
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,16,4,128,0,1,float16,float16,0,0.20889600118001303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,16,8,128,0,1,float16,float16,0,0.2126506765683492
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,16,16,128,0,1,float16,float16,0,0.1295360028743744
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,16,1,128,0,1,float16,float16,0,0.11741866668065389
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,16,2,128,0,1,float16,float16,0,0.11690666278203328
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,16,1,128,0,1,float16,float16,0,0.07526400188604991
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,16,16,128,0,1,float16,float16,0,0.07679999868075053
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,16,4,128,0,1,float16,float16,0,0.1186133325099945
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,16,2,128,0,1,float16,float16,0,0.07543466488520305
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,16,4,128,0,1,float16,float16,0,0.07509333391984303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,16,8,128,0,1,float16,float16,0,0.07560533285140991
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,16,8,128,0,1,float16,float16,0,0.12219732999801636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,16,1,128,0,1,float16,float16,0,1.827840010325114
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,16,2,128,0,1,float16,float16,0,1.872554620107015
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,16,4,128,0,1,float16,float16,0,1.9198293685913086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,512,16,1,128,0,1,float16,float16,0,3.6565332412719727
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,512,16,2,128,0,1,float16,float16,0,3.7573973337809243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,16,1,128,0,1,float16,float16,0,0.9291093349456787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,512,16,4,128,0,1,float16,float16,0,3.8662827809651694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,16,16,128,0,1,float16,float16,0,2.2860800425211587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,16,16,128,0,1,float16,float16,0,1.1473920345306396
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,16,2,128,0,1,float16,float16,0,0.9471999804178873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,16,8,128,0,1,float16,float16,0,1.9978240331013997
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,16,4,128,0,1,float16,float16,0,0.9687039852142334
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,16,1,128,0,1,float16,float16,0,0.48076800505320233
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,16,2,128,0,1,float16,float16,0,0.4870826800664266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,16,4,128,0,1,float16,float16,0,0.49561599890391034
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,16,1,128,0,1,float16,float16,0,0.2524159948031108
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,16,16,128,0,1,float16,float16,0,0.5886293252309164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,16,2,128,0,1,float16,float16,0,0.2558293342590332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,512,16,8,128,0,1,float16,float16,0,4.100607872009277
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,16,16,128,0,1,float16,float16,0,0.3051519989967346
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,16,8,128,0,1,float16,float16,0,1.0018133322397869
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,16,8,128,0,1,float16,float16,0,0.5150719881057739
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,16,1,128,0,1,float16,float16,0,0.13875200351079306
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,16,2,128,0,1,float16,float16,0,0.14216533303260803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,16,4,128,0,1,float16,float16,0,0.265557328859965
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,16,16,128,0,1,float16,float16,0,0.16503467162450156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,16,8,128,0,1,float16,float16,0,0.2698240081469218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,16,4,128,0,1,float16,float16,0,0.14404267072677612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,16,16,128,0,1,float16,float16,0,0.09079466263453166
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,16,1,128,0,1,float16,float16,0,0.07884799937407176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,16,2,128,0,1,float16,float16,0,0.07935999830563863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,16,1,128,0,1,float16,float16,0,0.05358933409055074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,16,8,128,0,1,float16,float16,0,0.08294400076071422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,16,16,128,0,1,float16,float16,0,0.05444266895453135
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,16,2,128,0,1,float16,float16,0,0.052906667192777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,16,4,128,0,1,float16,float16,0,0.05273599922657013
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,16,4,128,0,1,float16,float16,0,0.08123733103275299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,16,8,128,0,1,float16,float16,0,0.053930665055910744
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,16,16,128,0,1,float16,float16,0,0.04027733455101649
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,16,1,128,0,1,float16,float16,0,0.03857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,16,2,128,0,1,float16,float16,0,0.03908266623814901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,16,8,128,0,1,float16,float16,0,0.14984533190727234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,16,4,128,0,1,float16,float16,0,0.03925333420435587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,16,8,128,0,1,float16,float16,0,0.039594667653242745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,16,2,128,0,1,float16,float16,0,0.7294293244679769
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,16,4,128,0,1,float16,float16,0,0.749397357304891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,256,16,1,128,0,1,float16,float16,0,1.4074880282084148
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,16,1,128,0,1,float16,float16,0,0.711850643157959
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,256,16,2,128,0,1,float16,float16,0,1.460053284962972
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,16,1,128,0,1,float16,float16,0,0.3677866856257121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,16,16,128,0,1,float16,float16,0,0.9562453428904215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,256,16,4,128,0,1,float16,float16,0,1.5073280334472656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,16,4,128,0,1,float16,float16,0,0.38656000296274823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,16,16,128,0,1,float16,float16,0,0.490666667620341
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,16,2,128,0,1,float16,float16,0,0.37495466073354083
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,16,8,128,0,1,float16,float16,0,0.7871146996816
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,256,16,8,128,0,1,float16,float16,0,1.5938560167948406
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,16,1,128,0,1,float16,float16,0,0.19473065932591757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,16,16,128,0,1,float16,float16,0,0.26146133740743
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,16,2,128,0,1,float16,float16,0,0.19729065895080566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,16,8,128,0,1,float16,float16,0,0.40243200461069745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,16,4,128,0,1,float16,float16,0,0.2039466698964437
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,16,1,128,0,1,float16,float16,0,0.10734933614730835
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,16,2,128,0,1,float16,float16,0,0.13209600249926248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,16,16,128,0,1,float16,float16,0,0.1430186629295349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,16,4,128,0,1,float16,float16,0,0.1114453375339508
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,16,1,128,0,1,float16,float16,0,0.05922133227189382
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,16,8,128,0,1,float16,float16,0,0.1181013286113739
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,16,16,128,0,1,float16,float16,0,0.07526400188604991
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,16,8,128,0,1,float16,float16,0,0.21401600042978922
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,16,2,128,0,1,float16,float16,0,0.06058666606744131
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,16,4,128,0,1,float16,float16,0,0.061610668897628784
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,16,1,128,0,1,float16,float16,0,0.03976533313592275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,16,8,128,0,1,float16,float16,0,0.06519466638565063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,16,2,128,0,1,float16,float16,0,0.039936001102129616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,16,16,128,0,1,float16,float16,0,0.04232533276081085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,16,4,128,0,1,float16,float16,0,0.05273599922657013
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,16,1,128,0,1,float16,float16,0,0.03054933249950409
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,16,16,128,0,1,float16,float16,0,0.03259733319282532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,16,8,128,0,1,float16,float16,0,0.040618665516376495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,16,2,128,0,1,float16,float16,0,0.031231999397277832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,16,4,128,0,1,float16,float16,0,0.0314026673634847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,16,8,128,0,1,float16,float16,0,0.031231999397277832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,16,1,128,0,1,float16,float16,0,0.023039999107519787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,16,16,128,0,1,float16,float16,0,0.023210667073726654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,16,2,128,0,1,float16,float16,0,0.023210667073726654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,16,4,128,0,1,float16,float16,0,0.023210667073726654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,16,8,128,0,1,float16,float16,0,0.023039999107519787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,16,1,128,0,1,float16,float16,0,0.31573333342870075
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,16,2,128,0,1,float16,float16,0,0.32409600416819256
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,128,16,1,128,0,1,float16,float16,0,0.6132053136825562
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,128,16,2,128,0,1,float16,float16,0,0.6333440144856771
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,128,16,4,128,0,1,float16,float16,0,0.6574079990386963
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,16,16,128,0,1,float16,float16,0,0.4648960034052531
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,16,1,128,0,1,float16,float16,0,0.16691199938456217
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,16,4,128,0,1,float16,float16,0,0.3351893424987793
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,128,16,8,128,0,1,float16,float16,0,0.7113386789957682
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,16,16,128,0,1,float16,float16,0,0.24268800020217896
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,16,2,128,0,1,float16,float16,0,0.17151999473571777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,16,8,128,0,1,float16,float16,0,0.3638613224029541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,16,4,128,0,1,float16,float16,0,0.1776640017827352
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,16,16,128,0,1,float16,float16,0,0.13346133629480997
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,16,8,128,0,1,float16,float16,0,0.1962666710217794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,16,1,128,0,1,float16,float16,0,0.08925867080688477
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,16,2,128,0,1,float16,float16,0,0.09267200032869975
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,16,4,128,0,1,float16,float16,0,0.09676800171534221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,16,8,128,0,1,float16,float16,0,0.10547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,16,16,128,0,1,float16,float16,0,0.06809600194295247
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,16,1,128,0,1,float16,float16,0,0.04863999783992767
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,16,2,128,0,1,float16,float16,0,0.05000533163547516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,16,4,128,0,1,float16,float16,0,0.05085866649945577
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,16,1,128,0,1,float16,float16,0,0.03362133353948593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,16,16,128,0,1,float16,float16,0,0.03703466554482778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,16,2,128,0,1,float16,float16,0,0.034304000437259674
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,16,4,128,0,1,float16,float16,0,0.03515733281771342
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,16,8,128,0,1,float16,float16,0,0.03498666733503342
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,16,16,128,0,1,float16,float16,0,0.0264533335963885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,16,8,128,0,1,float16,float16,0,0.053247998158137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,16,1,128,0,1,float16,float16,0,0.025087999800841015
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,16,2,128,0,1,float16,float16,0,0.025258667767047882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,16,4,128,0,1,float16,float16,0,0.025770666698614757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,16,8,128,0,1,float16,float16,0,0.025600001215934753
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,16,16,128,0,1,float16,float16,0,0.018602666755517323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,16,1,128,0,1,float16,float16,0,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,16,2,128,0,1,float16,float16,0,0.018432000031073887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,16,8,128,0,1,float16,float16,0,0.01877333347996076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,16,16,128,0,1,float16,float16,0,0.017237332959969837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,16,1,128,0,1,float16,float16,0,0.017237332959969837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,16,4,128,0,1,float16,float16,0,0.018432000031073887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,16,4,128,0,1,float16,float16,0,0.017237332959969837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,16,2,128,0,1,float16,float16,0,0.017407999684413273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,16,8,128,0,1,float16,float16,0,0.017407999684413273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,64,16,1,128,0,1,float16,float16,0,0.3099306623140971
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,64,16,2,128,0,1,float16,float16,0,0.32051199674606323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,16,1,128,0,1,float16,float16,0,0.16691199938456217
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,16,2,128,0,1,float16,float16,0,0.1718613306681315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,16,4,128,0,1,float16,float16,0,0.1795413295427958
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,64,16,4,128,0,1,float16,float16,0,0.33297065893809
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,64,16,8,128,0,1,float16,float16,0,0.3689813216527303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,16,16,128,0,1,float16,float16,0,0.24439465999603271
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,16,1,128,0,1,float16,float16,0,0.08891733487447102
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,16,2,128,0,1,float16,float16,0,0.09284266829490662
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,16,8,128,0,1,float16,float16,0,0.19660800695419312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,16,16,128,0,1,float16,float16,0,0.13363200426101685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,16,4,128,0,1,float16,float16,0,0.10052266716957092
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,16,8,128,0,1,float16,float16,0,0.1109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,16,2,128,0,1,float16,float16,0,0.051029334465662636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,16,16,128,0,1,float16,float16,0,0.06894933183987935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,16,4,128,0,1,float16,float16,0,0.051541333397229515
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,16,8,128,0,1,float16,float16,0,0.05478399991989136
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,16,1,128,0,1,float16,float16,0,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,16,1,128,0,1,float16,float16,0,0.03498666733503342
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,16,16,128,0,1,float16,float16,0,0.03737599899371465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,16,2,128,0,1,float16,float16,0,0.03549866626660029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,16,4,128,0,1,float16,float16,0,0.03601066768169403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,16,8,128,0,1,float16,float16,0,0.0363520011305809
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,16,1,128,0,1,float16,float16,0,0.024234667420387268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,16,16,128,0,1,float16,float16,0,0.025429333249727886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,16,4,128,0,1,float16,float16,0,0.024746666351954143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,16,8,128,0,1,float16,float16,0,0.025258667767047882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,16,1,128,0,1,float16,float16,0,0.018432000031073887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,16,2,128,0,1,float16,float16,0,0.024234667420387268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,16,16,128,0,1,float16,float16,0,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,16,2,128,0,1,float16,float16,0,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,16,4,128,0,1,float16,float16,0,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,16,16,128,0,1,float16,float16,0,0.01672533278663953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,16,8,128,0,1,float16,float16,0,0.018602666755517323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,16,1,128,0,1,float16,float16,0,0.0170666662355264
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,16,2,128,0,1,float16,float16,0,0.017237332959969837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,16,1,128,0,1,float16,float16,0,0.016554666062196095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,16,16,128,0,1,float16,float16,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,16,2,128,0,1,float16,float16,0,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,16,8,128,0,1,float16,float16,0,0.017237332959969837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,16,4,128,0,1,float16,float16,0,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,16,8,128,0,1,float16,float16,0,0.016554666062196095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,16,4,128,0,1,float16,float16,0,0.016895999511082966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,32,16,1,128,0,1,float16,float16,0,0.18295466899871826
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,16,1,128,0,1,float16,float16,0,0.09591466188430786
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,32,16,2,128,0,1,float16,float16,0,0.1868799924850464
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,16,2,128,0,1,float16,float16,0,0.09949866930643718
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,16,16,128,0,1,float16,float16,0,0.13482667009035745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,32,16,8,128,0,1,float16,float16,0,0.20770132541656494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,16,4,128,0,1,float16,float16,0,0.10547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,32,16,4,128,0,1,float16,float16,0,0.19319466749827066
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,16,8,128,0,1,float16,float16,0,0.11417599519093831
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,16,1,128,0,1,float16,float16,0,0.05017599960168203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,16,2,128,0,1,float16,float16,0,0.051541333397229515
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,16,16,128,0,1,float16,float16,0,0.07133866846561432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,16,4,128,0,1,float16,float16,0,0.05256533126036326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,16,8,128,0,1,float16,float16,0,0.05597866574923197
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,16,1,128,0,1,float16,float16,0,0.03362133353948593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,16,2,128,0,1,float16,float16,0,0.034645333886146545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,16,4,128,0,1,float16,float16,0,0.034645333886146545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,16,16,128,0,1,float16,float16,0,0.025941332181294758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,16,8,128,0,1,float16,float16,0,0.03532800078392029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,16,1,128,0,1,float16,float16,0,0.025600001215934753
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,16,16,128,0,1,float16,float16,0,0.03583999971548716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,16,2,128,0,1,float16,float16,0,0.025600001215934753
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,16,4,128,0,1,float16,float16,0,0.025941332181294758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,16,8,128,0,1,float16,float16,0,0.02628266563018163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,16,1,128,0,1,float16,float16,0,0.016895999511082966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,16,16,128,0,1,float16,float16,0,0.0170666662355264
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,16,2,128,0,1,float16,float16,0,0.0170666662355264
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,16,4,128,0,1,float16,float16,0,0.017237332959969837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,16,16,128,0,1,float16,float16,0,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,16,2,128,0,1,float16,float16,0,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,16,1,128,0,1,float16,float16,0,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,16,4,128,0,1,float16,float16,0,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,16,16,128,0,1,float16,float16,0,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,16,1,128,0,1,float16,float16,0,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,16,8,128,0,1,float16,float16,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,16,8,128,0,1,float16,float16,0,0.017407999684413273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,16,2,128,0,1,float16,float16,0,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,16,4,128,0,1,float16,float16,0,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,16,8,128,0,1,float16,float16,0,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,16,16,128,0,1,float16,float16,0,0.013823999712864557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,16,1,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,16,2,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,16,4,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,16,16,1,128,0,1,float16,float16,0,0.13926399747530618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,16,8,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,16,16,2,128,0,1,float16,float16,0,0.1430186629295349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,16,1,128,0,1,float16,float16,0,0.07202133536338806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,16,16,4,128,0,1,float16,float16,0,0.14813866217931113
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,16,16,8,128,0,1,float16,float16,0,0.15598932902018228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,16,2,128,0,1,float16,float16,0,0.0730453332265218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,16,16,128,0,1,float16,float16,0,0.0865280032157898
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,16,4,128,0,1,float16,float16,0,0.07423999905586243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,16,16,128,0,1,float16,float16,0,0.04369066655635834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,16,1,128,0,1,float16,float16,0,0.042837331692377724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,16,4,128,0,1,float16,float16,0,0.044031997521718345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,16,2,128,0,1,float16,float16,0,0.04351999859015147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,16,8,128,0,1,float16,float16,0,0.04437333345413208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,16,16,128,0,1,float16,float16,0,0.02935466667016347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,16,1,128,0,1,float16,float16,0,0.029696000119050343
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,16,8,128,0,1,float16,float16,0,0.07799466451009114
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,16,2,128,0,1,float16,float16,0,0.04351999859015147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,16,4,128,0,1,float16,float16,0,0.030378667016824085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,16,1,128,0,1,float16,float16,0,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,16,8,128,0,1,float16,float16,0,0.03054933249950409
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,16,2,128,0,1,float16,float16,0,0.021162666380405426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,16,4,128,0,1,float16,float16,0,0.021162666380405426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,16,16,128,0,1,float16,float16,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,16,1,128,0,1,float16,float16,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,16,8,128,0,1,float16,float16,0,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,16,4,128,0,1,float16,float16,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,16,16,128,0,1,float16,float16,0,0.020309332758188248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,16,16,128,0,1,float16,float16,0,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,16,8,128,0,1,float16,float16,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,16,1,128,0,1,float16,float16,0,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,16,2,128,0,1,float16,float16,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,16,2,128,0,1,float16,float16,0,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,16,8,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,16,1,128,0,1,float16,float16,0,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,16,2,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,16,4,128,0,1,float16,float16,0,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,16,4,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,16,16,128,0,1,float16,float16,0,0.013653332988421122
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,16,16,128,0,1,float16,float16,0,0.013823999712864557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,16,8,128,0,1,float16,float16,0,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,16,1,128,0,1,float16,float16,0,0.013823999712864557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,16,2,128,0,1,float16,float16,0,0.013823999712864557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,16,4,128,0,1,float16,float16,0,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,16,8,128,0,1,float16,float16,0,0.013823999712864557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,12,1,128,0,1,float16,float16,0,22.401878356933594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,12,2,128,0,1,float16,float16,0,22.27814483642578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,12,12,128,0,1,float16,float16,0,11.410261789957682
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,12,4,128,0,1,float16,float16,0,22.321322123209637
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,12,1,128,0,1,float16,float16,0,11.454634348551432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,12,2,128,0,1,float16,float16,0,11.208021799723307
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16384,12,1,128,0,1,float16,float16,0,43.624104817708336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,12,12,128,0,1,float16,float16,0,22.255958557128906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,12,12,128,0,1,float16,float16,0,5.866325378417969
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16384,12,2,128,0,1,float16,float16,0,44.41156514485677
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,12,1,128,0,1,float16,float16,0,5.827925364176433
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,12,4,128,0,1,float16,float16,0,11.353599548339844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,12,2,128,0,1,float16,float16,0,5.7811628977457685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,12,4,128,0,1,float16,float16,0,5.713066736857097
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16384,12,4,128,0,1,float16,float16,0,44.490753173828125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,12,2,128,0,1,float16,float16,0,12.709205627441406
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,12,1,128,0,1,float16,float16,0,12.855978647867838
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,12288,12,1,128,0,1,float16,float16,0,25.04584503173828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,12,12,128,0,1,float16,float16,0,6.544384002685547
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,12,4,128,0,1,float16,float16,0,12.47863515218099
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,12288,12,2,128,0,1,float16,float16,0,25.15968068440755
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,12,12,128,0,1,float16,float16,0,12.978516896565756
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,12,1,128,0,1,float16,float16,0,6.465536117553711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,12,1,128,0,1,float16,float16,0,3.3401174545288086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,12,2,128,0,1,float16,float16,0,6.525781631469727
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,12288,12,4,128,0,1,float16,float16,0,25.4564692179362
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,12,4,128,0,1,float16,float16,0,6.368938446044922
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,12,12,128,0,1,float16,float16,0,3.375957489013672
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,12,2,128,0,1,float16,float16,0,3.3882452646891275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,12,4,128,0,1,float16,float16,0,3.363840103149414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,12,1,128,0,1,float16,float16,0,8.873984018961588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,12,2,128,0,1,float16,float16,0,8.98798942565918
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,12,12,128,0,1,float16,float16,0,4.607317288716634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,10240,12,1,128,0,1,float16,float16,0,17.629525502522785
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,12,4,128,0,1,float16,float16,0,8.994474411010742
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,12,1,128,0,1,float16,float16,0,4.476928075154622
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,10240,12,2,128,0,1,float16,float16,0,17.504596710205078
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,12,2,128,0,1,float16,float16,0,4.525055885314941
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,12,12,128,0,1,float16,float16,0,9.073322931925455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,12,1,128,0,1,float16,float16,0,2.4246613184611
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,12,12,128,0,1,float16,float16,0,2.4308053652445474
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,12,4,128,0,1,float16,float16,0,4.597418785095215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,12,2,128,0,1,float16,float16,0,2.407935937245687
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,12,4,128,0,1,float16,float16,0,2.4041813214619956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,10240,12,4,128,0,1,float16,float16,0,17.898155212402344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,12,1,128,0,1,float16,float16,0,11.618816375732422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,12,2,128,0,1,float16,float16,0,11.402751922607422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,12,12,128,0,1,float16,float16,0,5.930837631225586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,12,4,128,0,1,float16,float16,0,11.315711975097656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,8192,12,1,128,0,1,float16,float16,0,23.036585489908855
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,12,1,128,0,1,float16,float16,0,5.83458137512207
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,8192,12,2,128,0,1,float16,float16,0,23.151105244954426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,12,2,128,0,1,float16,float16,0,5.725866953531901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,12,1,128,0,1,float16,float16,0,2.975402514139811
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,12,2,128,0,1,float16,float16,0,2.9851306279500327
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,12,12,128,0,1,float16,float16,0,11.684693654378256
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,12,12,128,0,1,float16,float16,0,2.9928105672200522
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,12,12,128,0,1,float16,float16,0,1.617919921875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,12,4,128,0,1,float16,float16,0,2.9984426498413086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,12,1,128,0,1,float16,float16,0,1.6092160542805989
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,12,4,128,0,1,float16,float16,0,5.816661198933919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,8192,12,4,128,0,1,float16,float16,0,23.197865804036457
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,12,2,128,0,1,float16,float16,0,1.624575932820638
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,12,4,128,0,1,float16,float16,0,1.6203093528747559
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,12,2,128,0,1,float16,float16,0,6.623402913411458
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,12,1,128,0,1,float16,float16,0,6.7203413645426435
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,12,12,128,0,1,float16,float16,0,3.3846613566080728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,12,4,128,0,1,float16,float16,0,6.762837092081706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,6144,12,1,128,0,1,float16,float16,0,12.900010426839193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,6144,12,2,128,0,1,float16,float16,0,13.06777572631836
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,12,1,128,0,1,float16,float16,0,3.326122601826986
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,12,12,128,0,1,float16,float16,0,6.813184102376302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,12,1,128,0,1,float16,float16,0,1.7582079569498699
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,12,12,128,0,1,float16,float16,0,1.77237335840861
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,6144,12,4,128,0,1,float16,float16,0,13.435221354166666
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,12,4,128,0,1,float16,float16,0,3.3356800079345703
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,12,2,128,0,1,float16,float16,0,3.343189239501953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,12,2,128,0,1,float16,float16,0,1.7561599413553874
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,12,4,128,0,1,float16,float16,0,1.7728853225708008
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,12,12,128,0,1,float16,float16,0,0.9886720180511475
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,12,1,128,0,1,float16,float16,0,0.9809919993082682
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,12,2,128,0,1,float16,float16,0,0.986794630686442
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,12,4,128,0,1,float16,float16,0,0.9886720180511475
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,12,1,128,0,1,float16,float16,0,6.184106826782227
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,12,2,128,0,1,float16,float16,0,6.063615798950195
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,12,12,128,0,1,float16,float16,0,3.1740585962931314
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,12,4,128,0,1,float16,float16,0,6.267904281616211
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,12,1,128,0,1,float16,float16,0,3.0445226033528647
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,4096,12,1,128,0,1,float16,float16,0,12.221268971761068
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,12,2,128,0,1,float16,float16,0,3.0653438568115234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,12,1,128,0,1,float16,float16,0,1.5764479637145996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,4096,12,2,128,0,1,float16,float16,0,12.408490498860678
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,12,4,128,0,1,float16,float16,0,3.06551456451416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,12,2,128,0,1,float16,float16,0,1.5856639544169109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,12,12,128,0,1,float16,float16,0,1.6317440668741863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,12,12,128,0,1,float16,float16,0,6.362112045288086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,4096,12,4,128,0,1,float16,float16,0,12.268202463785807
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,12,4,128,0,1,float16,float16,0,1.5972693761189778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,12,1,128,0,1,float16,float16,0,0.4945919911066691
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,12,12,128,0,1,float16,float16,0,0.5046613216400146
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,12,4,128,0,1,float16,float16,0,0.8620373407999674
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,12,12,128,0,1,float16,float16,0,0.8753493626912435
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,12,2,128,0,1,float16,float16,0,0.49800535043080646
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,12,2,128,0,1,float16,float16,0,0.8552107016245524
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,12,1,128,0,1,float16,float16,0,0.8637440204620361
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,12,4,128,0,1,float16,float16,0,0.5034666856129965
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,12,1,128,0,1,float16,float16,0,3.5686400731404624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,12,2,128,0,1,float16,float16,0,3.5945812861124673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,12,12,128,0,1,float16,float16,0,1.8943999608357747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,12,4,128,0,1,float16,float16,0,3.6326398849487305
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,3072,12,1,128,0,1,float16,float16,0,7.260501225789388
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,12,1,128,0,1,float16,float16,0,1.800704002380371
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,3072,12,2,128,0,1,float16,float16,0,7.307946523030599
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,12,12,128,0,1,float16,float16,0,3.7143894831339517
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,12,2,128,0,1,float16,float16,0,1.803434689839681
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,12,1,128,0,1,float16,float16,0,0.9618773460388184
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,12,2,128,0,1,float16,float16,0,0.9531733194986979
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,3072,12,4,128,0,1,float16,float16,0,7.2618668874104815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,12,4,128,0,1,float16,float16,0,1.8249386151631672
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,12,12,128,0,1,float16,float16,0,0.9914027055104574
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,12,4,128,0,1,float16,float16,0,0.9598293304443359
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,12,12,128,0,1,float16,float16,0,0.5413546562194824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,12,2,128,0,1,float16,float16,0,0.5338453451792399
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,12,1,128,0,1,float16,float16,0,0.31436800956726074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,12,4,128,0,1,float16,float16,0,0.5246293147404989
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,12,12,128,0,1,float16,float16,0,0.3188053369522095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,12,1,128,0,1,float16,float16,0,0.5312853256861368
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,12,4,128,0,1,float16,float16,0,0.31436800956726074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,12,2,128,0,1,float16,float16,0,0.3123199939727783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,12,1,128,0,1,float16,float16,0,3.428863843282064
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,12,2,128,0,1,float16,float16,0,3.4751148223876953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,12,4,128,0,1,float16,float16,0,3.485525449117025
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,12,1,128,0,1,float16,float16,0,1.719637393951416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,2048,12,1,128,0,1,float16,float16,0,6.939818700154622
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,12,2,128,0,1,float16,float16,0,1.7309014002482097
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,12,12,128,0,1,float16,float16,0,1.8406400680541992
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,2048,12,2,128,0,1,float16,float16,0,7.039829254150391
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,12,12,128,0,1,float16,float16,0,3.6418558756510415
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,12,1,128,0,1,float16,float16,0,0.8878080050150553
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,12,2,128,0,1,float16,float16,0,0.8886613051096598
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,12,4,128,0,1,float16,float16,0,0.8980480035146078
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,12,12,128,0,1,float16,float16,0,0.9424213568369547
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,12,4,128,0,1,float16,float16,0,1.7580374081929524
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,12,1,128,0,1,float16,float16,0,0.4795733292897542
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,12,12,128,0,1,float16,float16,0,0.5109759966532389
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,12,2,128,0,1,float16,float16,0,0.4804266691207886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,2048,12,4,128,0,1,float16,float16,0,7.023445129394531
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,12,4,128,0,1,float16,float16,0,0.4828159809112549
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,12,2,128,0,1,float16,float16,0,0.279039998849233
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,12,1,128,0,1,float16,float16,0,0.27613866329193115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,12,1,128,0,1,float16,float16,0,0.17834667364756265
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,12,4,128,0,1,float16,float16,0,0.2786986629168193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,12,12,128,0,1,float16,float16,0,0.18107734123865762
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,12,12,128,0,1,float16,float16,0,0.2872320016225179
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,12,4,128,0,1,float16,float16,0,0.1790293256441752
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,12,2,128,0,1,float16,float16,0,0.1781760056813558
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,12,1,128,0,1,float16,float16,0,2.1101226806640625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,12,2,128,0,1,float16,float16,0,2.106368064880371
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,12,12,128,0,1,float16,float16,0,1.1533652941385906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,12,4,128,0,1,float16,float16,0,2.135039965311686
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,12,1,128,0,1,float16,float16,0,1.060863971710205
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1536,12,1,128,0,1,float16,float16,0,4.1965227127075195
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1536,12,2,128,0,1,float16,float16,0,4.211882591247559
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,12,12,128,0,1,float16,float16,0,2.255359967549642
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,12,1,128,0,1,float16,float16,0,0.5550080140431722
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,12,2,128,0,1,float16,float16,0,0.5556906859079996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,12,12,128,0,1,float16,float16,0,0.59443199634552
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,12,4,128,0,1,float16,float16,0,1.0804906686147053
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1536,12,4,128,0,1,float16,float16,0,4.278784116109212
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,12,4,128,0,1,float16,float16,0,0.5604693492253622
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,12,2,128,0,1,float16,float16,0,1.0641067028045654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,12,1,128,0,1,float16,float16,0,0.30242133140563965
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,12,2,128,0,1,float16,float16,0,0.3063466747601827
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,12,4,128,0,1,float16,float16,0,0.31112533807754517
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,12,12,128,0,1,float16,float16,0,0.32307199637095135
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,12,1,128,0,1,float16,float16,0,0.17732266585032144
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,12,2,128,0,1,float16,float16,0,0.17459199825922647
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,12,12,128,0,1,float16,float16,0,0.18517333269119263
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,12,12,128,0,1,float16,float16,0,0.12066133817036946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,12,1,128,0,1,float16,float16,0,0.11997866630554199
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,12,4,128,0,1,float16,float16,0,0.12151466806729634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,12,2,128,0,1,float16,float16,0,0.12083199620246887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,12,4,128,0,1,float16,float16,0,0.17732266585032144
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,12,1,128,0,1,float16,float16,0,2.0285439491271973
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,12,2,128,0,1,float16,float16,0,2.0725760459899902
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,12,4,128,0,1,float16,float16,0,2.094933350880941
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,1024,12,1,128,0,1,float16,float16,0,4.056405385335286
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,12,1,128,0,1,float16,float16,0,1.0308266480763753
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,12,12,128,0,1,float16,float16,0,1.1707733472188313
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,12,2,128,0,1,float16,float16,0,1.042090654373169
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,1024,12,2,128,0,1,float16,float16,0,4.085930824279785
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,12,1,128,0,1,float16,float16,0,0.5362346569697062
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,12,4,128,0,1,float16,float16,0,1.0610346794128418
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,12,12,128,0,1,float16,float16,0,2.3744853337605796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,12,2,128,0,1,float16,float16,0,0.5401599804560343
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,12,12,128,0,1,float16,float16,0,0.32358400026957196
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,12,4,128,0,1,float16,float16,0,0.5510826508204142
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,12,12,128,0,1,float16,float16,0,0.6068906784057617
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,12,1,128,0,1,float16,float16,0,0.28842665751775104
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,1024,12,4,128,0,1,float16,float16,0,4.195498784383138
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,12,2,128,0,1,float16,float16,0,0.29149866104125977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,12,4,128,0,1,float16,float16,0,0.2979840040206909
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,12,1,128,0,1,float16,float16,0,0.1616213321685791
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,12,2,128,0,1,float16,float16,0,0.1628159979979197
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,12,1,128,0,1,float16,float16,0,0.10035199920336406
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,12,12,128,0,1,float16,float16,0,0.10683733224868774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,12,2,128,0,1,float16,float16,0,0.0993280013402303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,12,4,128,0,1,float16,float16,0,0.16895999511082968
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,12,4,128,0,1,float16,float16,0,0.10154666503270467
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,12,12,128,0,1,float16,float16,0,0.06144000093142191
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,12,1,128,0,1,float16,float16,0,0.0602453351020813
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,12,2,128,0,1,float16,float16,0,0.05973333120346069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,12,4,128,0,1,float16,float16,0,0.06058666606744131
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,12,12,128,0,1,float16,float16,0,0.18295466899871826
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,12,1,128,0,1,float16,float16,0,1.3805227279663086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,12,2,128,0,1,float16,float16,0,1.4149972597757976
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,12,12,128,0,1,float16,float16,0,0.8569173018137614
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,12,4,128,0,1,float16,float16,0,1.4559574127197266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,512,12,1,128,0,1,float16,float16,0,2.7549012502034507
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,12,1,128,0,1,float16,float16,0,0.7033173243204752
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,12,2,128,0,1,float16,float16,0,0.7173120180765787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,12,1,128,0,1,float16,float16,0,0.3701759974161784
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,12,2,128,0,1,float16,float16,0,0.37273601690928143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,512,12,2,128,0,1,float16,float16,0,2.843306541442871
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,12,12,128,0,1,float16,float16,0,1.7254400253295898
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,12,12,128,0,1,float16,float16,0,0.4396373430887858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,12,1,128,0,1,float16,float16,0,0.19797333081563315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,512,12,4,128,0,1,float16,float16,0,2.940586725870768
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,12,4,128,0,1,float16,float16,0,0.3824640115102132
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,12,12,128,0,1,float16,float16,0,0.2315946618715922
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,12,4,128,0,1,float16,float16,0,0.7338666915893555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,12,4,128,0,1,float16,float16,0,0.20497065782546997
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,12,12,128,0,1,float16,float16,0,0.13243732849756876
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,12,2,128,0,1,float16,float16,0,0.20070399840672812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,12,1,128,0,1,float16,float16,0,0.10973866780598958
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,12,2,128,0,1,float16,float16,0,0.11246933539708455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,12,4,128,0,1,float16,float16,0,0.11639466881752014
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,12,12,128,0,1,float16,float16,0,0.07458133498827617
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,12,2,128,0,1,float16,float16,0,0.06741333504517873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,12,4,128,0,1,float16,float16,0,0.06911999980608623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,12,1,128,0,1,float16,float16,0,0.06809600194295247
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,12,1,128,0,1,float16,float16,0,0.03976533313592275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,12,2,128,0,1,float16,float16,0,0.039594667653242745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,12,1,128,0,1,float16,float16,0,0.032085334261258446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,12,12,128,0,1,float16,float16,0,0.03242666771014532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,12,4,128,0,1,float16,float16,0,0.040618665516376495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,12,12,128,0,1,float16,float16,0,0.04130133241415024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,12,2,128,0,1,float16,float16,0,0.032255999743938446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,12,4,128,0,1,float16,float16,0,0.032255999743938446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,12,1,128,0,1,float16,float16,0,0.5434026718139648
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,12,2,128,0,1,float16,float16,0,0.552618662516276
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,256,12,1,128,0,1,float16,float16,0,1.0627413590749104
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,12,4,128,0,1,float16,float16,0,0.5734399954477946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,256,12,2,128,0,1,float16,float16,0,1.0936319828033447
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,12,12,128,0,1,float16,float16,0,0.3752959966659546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,12,1,128,0,1,float16,float16,0,0.28381866216659546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,12,12,128,0,1,float16,float16,0,0.7226026852925619
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,12,2,128,0,1,float16,float16,0,0.28910932938257855
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,256,12,4,128,0,1,float16,float16,0,1.130837361017863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,12,1,128,0,1,float16,float16,0,0.15359999736150107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,12,12,128,0,1,float16,float16,0,0.20172800620396933
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,12,4,128,0,1,float16,float16,0,0.2991786599159241
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,12,4,128,0,1,float16,float16,0,0.16059733430544534
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,12,12,128,0,1,float16,float16,0,0.11485866705576579
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,12,2,128,0,1,float16,float16,0,0.15633066495259604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,12,2,128,0,1,float16,float16,0,0.08721066514650981
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,12,1,128,0,1,float16,float16,0,0.04983466863632202
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,12,4,128,0,1,float16,float16,0,0.09028266867001851
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,12,12,128,0,1,float16,float16,0,0.05751466751098633
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,12,2,128,0,1,float16,float16,0,0.0506879985332489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,12,1,128,0,1,float16,float16,0,0.08379733562469482
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,12,4,128,0,1,float16,float16,0,0.05171200136343638
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,12,1,128,0,1,float16,float16,0,0.031061333914597828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,12,2,128,0,1,float16,float16,0,0.0314026673634847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,12,4,128,0,1,float16,float16,0,0.032255999743938446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,12,1,128,0,1,float16,float16,0,0.024405332903067272
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,12,12,128,0,1,float16,float16,0,0.02491733431816101
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,12,12,128,0,1,float16,float16,0,0.03362133353948593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,12,4,128,0,1,float16,float16,0,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,12,12,128,0,1,float16,float16,0,0.022015998760859173
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,12,2,128,0,1,float16,float16,0,0.024234667420387268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,12,1,128,0,1,float16,float16,0,0.02218666672706604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,12,2,128,0,1,float16,float16,0,0.02218666672706604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,12,4,128,0,1,float16,float16,0,0.02218666672706604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,12,2,128,0,1,float16,float16,0,0.24524799982706705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,128,12,1,128,0,1,float16,float16,0,0.4602880080540975
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,12,1,128,0,1,float16,float16,0,0.2397866646448771
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,12,12,128,0,1,float16,float16,0,0.21009065707524618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,12,4,128,0,1,float16,float16,0,0.2648746569951375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,128,12,4,128,0,1,float16,float16,0,0.5017600059509277
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,12,12,128,0,1,float16,float16,0,0.35310932000478107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,128,12,2,128,0,1,float16,float16,0,0.47377065817515057
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,12,1,128,0,1,float16,float16,0,0.12970667084058127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,12,2,128,0,1,float16,float16,0,0.13431466619173685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,12,12,128,0,1,float16,float16,0,0.10274133086204529
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,12,1,128,0,1,float16,float16,0,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,12,2,128,0,1,float16,float16,0,0.07253333429495494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,12,4,128,0,1,float16,float16,0,0.13994666934013367
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,12,4,128,0,1,float16,float16,0,0.07850666840871175
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,12,1,128,0,1,float16,float16,0,0.040106666584809623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,12,2,128,0,1,float16,float16,0,0.04027733455101649
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,12,4,128,0,1,float16,float16,0,0.041984001795450844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,12,12,128,0,1,float16,float16,0,0.04744533201058706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,12,12,128,0,1,float16,float16,0,0.027647999425729115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,12,1,128,0,1,float16,float16,0,0.025600001215934753
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,12,12,128,0,1,float16,float16,0,0.020309332758188248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,12,4,128,0,1,float16,float16,0,0.0266239990790685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,12,2,128,0,1,float16,float16,0,0.025941332181294758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,12,1,128,0,1,float16,float16,0,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,12,2,128,0,1,float16,float16,0,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,12,1,128,0,1,float16,float16,0,0.017237332959969837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,12,12,128,0,1,float16,float16,0,0.017407999684413273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,12,2,128,0,1,float16,float16,0,0.01757866640885671
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,12,4,128,0,1,float16,float16,0,0.01757866640885671
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,12,4,128,0,1,float16,float16,0,0.019968000551064808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,12,1,128,0,1,float16,float16,0,0.01672533278663953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,12,2,128,0,1,float16,float16,0,0.01672533278663953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,12,12,128,0,1,float16,float16,0,0.01672533278663953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,12,4,128,0,1,float16,float16,0,0.01672533278663953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,64,12,1,128,0,1,float16,float16,0,0.23432532946268717
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,64,12,2,128,0,1,float16,float16,0,0.23893332481384277
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,12,2,128,0,1,float16,float16,0,0.130730668703715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,12,12,128,0,1,float16,float16,0,0.10410666465759277
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,12,4,128,0,1,float16,float16,0,0.14131200313568115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,64,12,4,128,0,1,float16,float16,0,0.2611200014750163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,12,12,128,0,1,float16,float16,0,0.18961066007614136
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,12,1,128,0,1,float16,float16,0,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,12,2,128,0,1,float16,float16,0,0.07048533360163371
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,12,1,128,0,1,float16,float16,0,0.12680533528327942
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,12,4,128,0,1,float16,float16,0,0.0769706666469574
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,12,2,128,0,1,float16,float16,0,0.039936001102129616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,12,12,128,0,1,float16,float16,0,0.045567999283472695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,12,1,128,0,1,float16,float16,0,0.03908266623814901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,12,1,128,0,1,float16,float16,0,0.025087999800841015
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,12,12,128,0,1,float16,float16,0,0.02611200014750163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,12,4,128,0,1,float16,float16,0,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,12,12,128,0,1,float16,float16,0,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,12,4,128,0,1,float16,float16,0,0.025770666698614757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,12,1,128,0,1,float16,float16,0,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,12,2,128,0,1,float16,float16,0,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,12,4,128,0,1,float16,float16,0,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,12,12,128,0,1,float16,float16,0,0.017237332959969837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,12,2,128,0,1,float16,float16,0,0.01757866640885671
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,12,4,128,0,1,float16,float16,0,0.01757866640885671
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,12,12,128,0,1,float16,float16,0,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,12,1,128,0,1,float16,float16,0,0.017237332959969837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,12,2,128,0,1,float16,float16,0,0.025600001215934753
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,12,2,128,0,1,float16,float16,0,0.01672533278663953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,12,4,128,0,1,float16,float16,0,0.016895999511082966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,12,1,128,0,1,float16,float16,0,0.016554666062196095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,12,12,128,0,1,float16,float16,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,12,1,128,0,1,float16,float16,0,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,12,2,128,0,1,float16,float16,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,12,4,128,0,1,float16,float16,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,32,12,2,128,0,1,float16,float16,0,0.14523733655611673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,12,1,128,0,1,float16,float16,0,0.07526400188604991
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,32,12,4,128,0,1,float16,float16,0,0.15172266960144043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,12,2,128,0,1,float16,float16,0,0.07850666840871175
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,12,4,128,0,1,float16,float16,0,0.08277333279450734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,32,12,1,128,0,1,float16,float16,0,0.14284800489743552
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,12,1,128,0,1,float16,float16,0,0.041984001795450844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,12,12,128,0,1,float16,float16,0,0.10444800059000652
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,12,2,128,0,1,float16,float16,0,0.042837331692377724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,12,1,128,0,1,float16,float16,0,0.0266239990790685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,12,12,128,0,1,float16,float16,0,0.027647999425729115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,12,12,128,0,1,float16,float16,0,0.048810665806134544
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,12,4,128,0,1,float16,float16,0,0.027647999425729115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,12,4,128,0,1,float16,float16,0,0.04351999859015147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,12,2,128,0,1,float16,float16,0,0.026965332527955372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,12,1,128,0,1,float16,float16,0,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,12,12,128,0,1,float16,float16,0,0.018602666755517323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,12,2,128,0,1,float16,float16,0,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,12,4,128,0,1,float16,float16,0,0.018602666755517323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,12,1,128,0,1,float16,float16,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,12,12,128,0,1,float16,float16,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,12,12,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,12,2,128,0,1,float16,float16,0,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,12,4,128,0,1,float16,float16,0,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,12,2,128,0,1,float16,float16,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,12,1,128,0,1,float16,float16,0,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,12,12,128,0,1,float16,float16,0,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,12,4,128,0,1,float16,float16,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,12,2,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,12,4,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,12,12,128,0,1,float16,float16,0,0.013823999712864557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,12,1,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,12,4,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,12,1,128,0,1,float16,float16,0,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,12,2,128,0,1,float16,float16,0,0.0290133332212766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,16,12,2,128,0,1,float16,float16,0,0.11076266566912334
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,12,2,128,0,1,float16,float16,0,0.05922133227189382
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,12,12,128,0,1,float16,float16,0,0.06417066852251689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,16,12,4,128,0,1,float16,float16,0,0.11622400085131328
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,12,1,128,0,1,float16,float16,0,0.058880001306533813
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,16,12,1,128,0,1,float16,float16,0,0.10769066214561462
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,12,1,128,0,1,float16,float16,0,0.03515733281771342
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,12,2,128,0,1,float16,float16,0,0.03566933423280716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,12,12,128,0,1,float16,float16,0,0.03515733281771342
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,12,4,128,0,1,float16,float16,0,0.03601066768169403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,12,12,128,0,1,float16,float16,0,0.022698665658632915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,12,4,128,0,1,float16,float16,0,0.060415998101234436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,12,4,128,0,1,float16,float16,0,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,12,1,128,0,1,float16,float16,0,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,12,12,128,0,1,float16,float16,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,12,2,128,0,1,float16,float16,0,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,12,4,128,0,1,float16,float16,0,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,12,2,128,0,1,float16,float16,0,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,12,1,128,0,1,float16,float16,0,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,12,1,128,0,1,float16,float16,0,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,12,4,128,0,1,float16,float16,0,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,12,12,128,0,1,float16,float16,0,0.013653332988421122
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,12,1,128,0,1,float16,float16,0,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,12,2,128,0,1,float16,float16,0,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,12,12,128,0,1,float16,float16,0,0.013653332988421122
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,12,1,128,0,1,float16,float16,0,0.013823999712864557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,12,12,128,0,1,float16,float16,0,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,12,4,128,0,1,float16,float16,0,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,12,2,128,0,1,float16,float16,0,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,12,2,128,0,1,float16,float16,0,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,12,4,128,0,1,float16,float16,0,0.013823999712864557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,12,1,128,0,1,float16,float16,0,0.013823999712864557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,12,2,128,0,1,float16,float16,0,0.013823999712864557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,12,12,128,0,1,float16,float16,0,0.01331199953953425
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,12,4,128,0,1,float16,float16,0,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,8,1,128,0,1,float16,float16,0,14.755840301513672
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,8,2,128,0,1,float16,float16,0,14.779221852620443
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,8,1,128,0,1,float16,float16,0,7.569237391153972
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,8,8,128,0,1,float16,float16,0,15.18933359781901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16384,8,1,128,0,1,float16,float16,0,29.853866577148438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,8,8,128,0,1,float16,float16,0,7.7460479736328125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,8,2,128,0,1,float16,float16,0,7.489023844401042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,8,4,128,0,1,float16,float16,0,15.035221099853516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16384,8,2,128,0,1,float16,float16,0,29.147476196289062
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,8,1,128,0,1,float16,float16,0,3.966634750366211
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,8,8,128,0,1,float16,float16,0,3.966634750366211
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16384,8,4,128,0,1,float16,float16,0,29.73730214436849
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,8,4,128,0,1,float16,float16,0,3.9565652211507163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,8,4,128,0,1,float16,float16,0,7.591936111450195
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,8,2,128,0,1,float16,float16,0,4.006229400634766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,8,1,128,0,1,float16,float16,0,8.692394892374674
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,8,2,128,0,1,float16,float16,0,8.451072057088217
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,12288,8,1,128,0,1,float16,float16,0,16.99840037027995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,8,4,128,0,1,float16,float16,0,8.514048258463541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,12288,8,2,128,0,1,float16,float16,0,16.557397206624348
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,8,1,128,0,1,float16,float16,0,4.3608748118082685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,8,8,128,0,1,float16,float16,0,8.637269337972006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,8,2,128,0,1,float16,float16,0,4.32861868540446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,8,8,128,0,1,float16,float16,0,4.396714528401692
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,8,1,128,0,1,float16,float16,0,2.3417174021402993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,12288,8,4,128,0,1,float16,float16,0,17.076736450195312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,8,8,128,0,1,float16,float16,0,2.3437652587890625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,8,2,128,0,1,float16,float16,0,2.3437652587890625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,8,4,128,0,1,float16,float16,0,2.3342080116271973
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,8,4,128,0,1,float16,float16,0,4.373504002888997
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,8,1,128,0,1,float16,float16,0,5.953194936116536
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,8,2,128,0,1,float16,float16,0,6.057984034220378
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,10240,8,1,128,0,1,float16,float16,0,11.76968510945638
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,8,4,128,0,1,float16,float16,0,6.080511728922526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,8,8,128,0,1,float16,float16,0,3.1230293909708657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,8,8,128,0,1,float16,float16,0,6.059861501057942
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,8,1,128,0,1,float16,float16,0,3.1008427937825522
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,8,2,128,0,1,float16,float16,0,3.0962346394856772
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,8,8,128,0,1,float16,float16,0,1.6965972582499187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,8,4,128,0,1,float16,float16,0,3.076266606648763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,10240,8,2,128,0,1,float16,float16,0,11.949396769205729
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,10240,8,4,128,0,1,float16,float16,0,11.692885080973307
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,8,1,128,0,1,float16,float16,0,1.6868693033854167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,8,2,128,0,1,float16,float16,0,1.6952320734659831
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,8,4,128,0,1,float16,float16,0,1.695573329925537
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,8,1,128,0,1,float16,float16,0,7.579647699991862
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,8,8,128,0,1,float16,float16,0,3.929941177368164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,8,2,128,0,1,float16,float16,0,7.781546910603841
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,8,4,128,0,1,float16,float16,0,7.799466451009114
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,8192,8,1,128,0,1,float16,float16,0,15.43185043334961
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,8,1,128,0,1,float16,float16,0,3.8828372955322266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,8192,8,2,128,0,1,float16,float16,0,15.519060770670572
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,8,2,128,0,1,float16,float16,0,3.8787412643432617
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,8,1,128,0,1,float16,float16,0,2.049877325693766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,8,8,128,0,1,float16,float16,0,2.0718933741251626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,8,8,128,0,1,float16,float16,0,7.842474619547526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,8,2,128,0,1,float16,float16,0,2.0519253412882485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,8,4,128,0,1,float16,float16,0,3.9000746409098306
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,8,1,128,0,1,float16,float16,0,1.1432960033416748
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,8,8,128,0,1,float16,float16,0,1.1521706581115723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,8,4,128,0,1,float16,float16,0,2.0604586601257324
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,8192,8,4,128,0,1,float16,float16,0,15.194965362548828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,8,2,128,0,1,float16,float16,0,1.1521706581115723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,8,4,128,0,1,float16,float16,0,1.1494399706522624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,8,1,128,0,1,float16,float16,0,4.4629332224528
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,8,2,128,0,1,float16,float16,0,4.419072151184082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,6144,8,1,128,0,1,float16,float16,0,8.672938664754232
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,8,4,128,0,1,float16,float16,0,4.447061220804851
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,8,1,128,0,1,float16,float16,0,2.268330732981364
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,8,8,128,0,1,float16,float16,0,4.481365203857422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,8,2,128,0,1,float16,float16,0,2.273962656656901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,6144,8,2,128,0,1,float16,float16,0,8.938496271769205
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,8,1,128,0,1,float16,float16,0,1.2212906678517659
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,8,8,128,0,1,float16,float16,0,2.319530646006266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,8,8,128,0,1,float16,float16,0,1.2327253023783367
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,6144,8,4,128,0,1,float16,float16,0,9.009152094523111
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,8,2,128,0,1,float16,float16,0,1.2301653226216633
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,8,4,128,0,1,float16,float16,0,2.2901760737101235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,8,4,128,0,1,float16,float16,0,1.2267519632975261
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,8,1,128,0,1,float16,float16,0,0.7248213291168213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,8,4,128,0,1,float16,float16,0,0.710314671198527
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,8,8,128,0,1,float16,float16,0,0.7039999961853027
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,8,2,128,0,1,float16,float16,0,0.7045119603474935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,8,1,128,0,1,float16,float16,0,4.099072138468425
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,8,2,128,0,1,float16,float16,0,4.101290702819824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,8,1,128,0,1,float16,float16,0,2.0671146710713706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,4096,8,1,128,0,1,float16,float16,0,8.32426643371582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,8,8,128,0,1,float16,float16,0,4.15829340616862
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,4096,8,2,128,0,1,float16,float16,0,8.216917037963867
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,8,8,128,0,1,float16,float16,0,2.1473280588785806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,8,2,128,0,1,float16,float16,0,2.0845227241516113
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,8,4,128,0,1,float16,float16,0,4.158464113871257
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,8,1,128,0,1,float16,float16,0,1.0912426312764485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,8,2,128,0,1,float16,float16,0,1.1033600171407063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,8,8,128,0,1,float16,float16,0,1.1205973625183105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,8,4,128,0,1,float16,float16,0,2.0976640383402505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,4096,8,4,128,0,1,float16,float16,0,8.216234842936197
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,8,8,128,0,1,float16,float16,0,0.6212266683578491
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,8,2,128,0,1,float16,float16,0,0.6027946472167969
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,8,4,128,0,1,float16,float16,0,0.6113280057907104
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,8,8,128,0,1,float16,float16,0,0.37717334429423016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,8,1,128,0,1,float16,float16,0,0.37034666538238525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,8,2,128,0,1,float16,float16,0,0.3653973340988159
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,8,1,128,0,1,float16,float16,0,0.6091093222300211
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,8,4,128,0,1,float16,float16,0,0.36983466148376465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,8,4,128,0,1,float16,float16,0,1.1064319610595703
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,8,1,128,0,1,float16,float16,0,2.4162987073262534
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,3072,8,1,128,0,1,float16,float16,0,4.753237406412761
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,8,4,128,0,1,float16,float16,0,2.4408747355143228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,8,2,128,0,1,float16,float16,0,2.4243200620015464
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,8,1,128,0,1,float16,float16,0,1.2313600381215413
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,8,8,128,0,1,float16,float16,0,2.4934399922688804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,3072,8,2,128,0,1,float16,float16,0,4.794879913330078
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,8,8,128,0,1,float16,float16,0,1.2960426807403564
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,8,2,128,0,1,float16,float16,0,1.2395520210266113
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,8,1,128,0,1,float16,float16,0,0.6654293139775594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,8,2,128,0,1,float16,float16,0,0.6669653256734213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,3072,8,4,128,0,1,float16,float16,0,4.937557220458984
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,8,4,128,0,1,float16,float16,0,1.2538879712422688
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,8,4,128,0,1,float16,float16,0,0.6765226523081461
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,8,8,128,0,1,float16,float16,0,0.688981294631958
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,8,1,128,0,1,float16,float16,0,0.3829760154088338
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,8,2,128,0,1,float16,float16,0,0.37649067242940265
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,8,8,128,0,1,float16,float16,0,0.24541866779327393
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,8,1,128,0,1,float16,float16,0,0.23995733261108398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,8,2,128,0,1,float16,float16,0,0.2379093368848165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,8,4,128,0,1,float16,float16,0,0.3802453279495239
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,8,8,128,0,1,float16,float16,0,0.3927040100097656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,8,4,128,0,1,float16,float16,0,0.24012800057729086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,8,1,128,0,1,float16,float16,0,2.315605322519938
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,8,2,128,0,1,float16,float16,0,2.3432532946268716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,2048,8,1,128,0,1,float16,float16,0,4.612266540527344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,8,1,128,0,1,float16,float16,0,1.1779413223266602
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,8,2,128,0,1,float16,float16,0,1.1893760363260906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,2048,8,2,128,0,1,float16,float16,0,4.6469119389851885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,8,8,128,0,1,float16,float16,0,2.4364372889200845
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,8,8,128,0,1,float16,float16,0,1.2477440039316814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,8,4,128,0,1,float16,float16,0,2.3753387133280435
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,8,1,128,0,1,float16,float16,0,0.6130346854527792
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,8,4,128,0,1,float16,float16,0,1.2093439896901448
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,8,2,128,0,1,float16,float16,0,0.6229333480199178
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,8,8,128,0,1,float16,float16,0,0.6483626763025919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,2048,8,4,128,0,1,float16,float16,0,4.776960055033366
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,8,4,128,0,1,float16,float16,0,0.6277120113372803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,8,2,128,0,1,float16,float16,0,0.3423573176066081
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,8,4,128,0,1,float16,float16,0,0.34781865278879803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,8,1,128,0,1,float16,float16,0,0.20138667027155557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,8,8,128,0,1,float16,float16,0,0.3573760191599528
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,8,8,128,0,1,float16,float16,0,0.20718934138615927
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,8,2,128,0,1,float16,float16,0,0.20360533396402994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,8,4,128,0,1,float16,float16,0,0.19950934251149496
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,8,1,128,0,1,float16,float16,0,0.34201598167419434
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,8,8,128,0,1,float16,float16,0,0.13482667009035745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,8,1,128,0,1,float16,float16,0,0.13312000036239624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,8,2,128,0,1,float16,float16,0,0.13636266191800436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,8,4,128,0,1,float16,float16,0,0.13294933239618936
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,8,1,128,0,1,float16,float16,0,1.4117546081542969
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,8,2,128,0,1,float16,float16,0,1.4346240361531575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1536,8,1,128,0,1,float16,float16,0,2.7861334482828775
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,8,4,128,0,1,float16,float16,0,1.4561279614766438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,8,1,128,0,1,float16,float16,0,0.7243093649546305
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1536,8,2,128,0,1,float16,float16,0,2.819925308227539
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,8,2,128,0,1,float16,float16,0,0.7335253556569418
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,8,8,128,0,1,float16,float16,0,0.7806293169657389
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,8,1,128,0,1,float16,float16,0,0.3848533233006795
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,8,8,128,0,1,float16,float16,0,0.41437868277231854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,8,8,128,0,1,float16,float16,0,1.5199573834737141
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1536,8,4,128,0,1,float16,float16,0,2.8654934565226235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,8,2,128,0,1,float16,float16,0,0.3911679983139038
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,8,4,128,0,1,float16,float16,0,0.7490560213724772
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,8,4,128,0,1,float16,float16,0,0.39458131790161133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,8,1,128,0,1,float16,float16,0,0.21708800395329794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,8,2,128,0,1,float16,float16,0,0.21845332781473795
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,8,8,128,0,1,float16,float16,0,0.14097066720326742
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,8,2,128,0,1,float16,float16,0,0.1365333298842112
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,8,8,128,0,1,float16,float16,0,0.2290346622467041
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,8,4,128,0,1,float16,float16,0,0.1365333298842112
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,8,1,128,0,1,float16,float16,0,0.1365333298842112
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,8,8,128,0,1,float16,float16,0,0.08533333738644917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,8,1,128,0,1,float16,float16,0,0.08447999755541484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,8,2,128,0,1,float16,float16,0,0.08482133348782857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,8,4,128,0,1,float16,float16,0,0.0846506655216217
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,8,4,128,0,1,float16,float16,0,0.22374399503072104
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,8,1,128,0,1,float16,float16,0,1.3730133374532063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,8,2,128,0,1,float16,float16,0,1.3844480514526367
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,8,4,128,0,1,float16,float16,0,1.4697813987731934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,8,1,128,0,1,float16,float16,0,0.7028053601582845
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,8,8,128,0,1,float16,float16,0,0.7958186467488607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,1024,8,1,128,0,1,float16,float16,0,2.6985813776652017
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,8,8,128,0,1,float16,float16,0,1.5738879839579265
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,8,2,128,0,1,float16,float16,0,0.7120213508605957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,1024,8,2,128,0,1,float16,float16,0,2.7386881510416665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,8,1,128,0,1,float16,float16,0,0.20684800545374551
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,8,8,128,0,1,float16,float16,0,0.41472001870473224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,8,4,128,0,1,float16,float16,0,0.38656000296274823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,8,8,128,0,1,float16,float16,0,0.22835199038187662
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,8,2,128,0,1,float16,float16,0,0.3763200044631958
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,8,1,128,0,1,float16,float16,0,0.3722240130106608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,8,4,128,0,1,float16,float16,0,0.7342080275217692
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,8,2,128,0,1,float16,float16,0,0.20753065745035806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,1024,8,4,128,0,1,float16,float16,0,2.860032081604004
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,8,1,128,0,1,float16,float16,0,0.11878400047620137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,8,4,128,0,1,float16,float16,0,0.21145600080490112
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,8,8,128,0,1,float16,float16,0,0.13004799683888754
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,8,8,128,0,1,float16,float16,0,0.07833600044250488
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,8,4,128,0,1,float16,float16,0,0.12219732999801636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,8,2,128,0,1,float16,float16,0,0.11895466844240825
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,8,1,128,0,1,float16,float16,0,0.07509333391984303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,8,2,128,0,1,float16,float16,0,0.07577600081761678
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,8,8,128,0,1,float16,float16,0,0.06075733403364817
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,8,1,128,0,1,float16,float16,0,0.05922133227189382
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,8,2,128,0,1,float16,float16,0,0.05973333120346069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,8,4,128,0,1,float16,float16,0,0.07526400188604991
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,8,4,128,0,1,float16,float16,0,0.05973333120346069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,8,1,128,0,1,float16,float16,0,0.934058666229248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,8,2,128,0,1,float16,float16,0,0.9598293304443359
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,8,8,128,0,1,float16,float16,0,0.582314650217692
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,8,1,128,0,1,float16,float16,0,0.48128000895182294
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,512,8,1,128,0,1,float16,float16,0,1.8452480634053547
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,8,4,128,0,1,float16,float16,0,1.0060799916585286
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,512,8,2,128,0,1,float16,float16,0,1.8903039296468098
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,8,2,128,0,1,float16,float16,0,0.4930560191472371
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,8,8,128,0,1,float16,float16,0,1.1494399706522624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,8,2,128,0,1,float16,float16,0,0.26077866554260254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,8,1,128,0,1,float16,float16,0,0.2539520064989726
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,8,8,128,0,1,float16,float16,0,0.30139732360839844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,8,4,128,0,1,float16,float16,0,0.27084799607594806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,8,1,128,0,1,float16,float16,0,0.14148267110188803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,8,8,128,0,1,float16,float16,0,0.16264533003171286
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,512,8,4,128,0,1,float16,float16,0,2.013696034749349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,8,4,128,0,1,float16,float16,0,0.5120000044504801
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,8,2,128,0,1,float16,float16,0,0.14250666896502176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,8,8,128,0,1,float16,float16,0,0.09250133236249287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,8,1,128,0,1,float16,float16,0,0.07867733140786488
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,8,2,128,0,1,float16,float16,0,0.08106666803359985
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,8,4,128,0,1,float16,float16,0,0.08379733562469482
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,8,8,128,0,1,float16,float16,0,0.05529599885145823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,8,4,128,0,1,float16,float16,0,0.14830933014551798
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,8,8,128,0,1,float16,float16,0,0.04027733455101649
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,8,4,128,0,1,float16,float16,0,0.05341866612434387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,8,1,128,0,1,float16,float16,0,0.03839999934037527
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,8,2,128,0,1,float16,float16,0,0.038912000755469
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,8,4,128,0,1,float16,float16,0,0.03942399968703588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,8,2,128,0,1,float16,float16,0,0.052906667192777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,8,8,128,0,1,float16,float16,0,0.030720000465710957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,8,1,128,0,1,float16,float16,0,0.031061333914597828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,8,1,128,0,1,float16,float16,0,0.05205333232879639
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,8,2,128,0,1,float16,float16,0,0.03089066594839096
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,8,4,128,0,1,float16,float16,0,0.03054933249950409
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,8,1,128,0,1,float16,float16,0,0.3710293372472127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,8,8,128,0,1,float16,float16,0,0.25890133778254193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,256,8,1,128,0,1,float16,float16,0,0.7215786774953207
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,8,4,128,0,1,float16,float16,0,0.4036266803741455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,256,8,2,128,0,1,float16,float16,0,0.7442773183186849
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,8,1,128,0,1,float16,float16,0,0.1962666710217794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,8,2,128,0,1,float16,float16,0,0.3829760154088338
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,8,8,128,0,1,float16,float16,0,0.4928853511810303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,8,2,128,0,1,float16,float16,0,0.20036266247431436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,8,1,128,0,1,float16,float16,0,0.10905599594116211
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,8,8,128,0,1,float16,float16,0,0.1431893308957418
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,256,8,4,128,0,1,float16,float16,0,0.7942826747894287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,8,4,128,0,1,float16,float16,0,0.21452800432840982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,8,2,128,0,1,float16,float16,0,0.11076266566912334
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,8,1,128,0,1,float16,float16,0,0.06007466713587443
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,8,2,128,0,1,float16,float16,0,0.06144000093142191
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,8,1,128,0,1,float16,float16,0,0.040106666584809623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,8,4,128,0,1,float16,float16,0,0.06519466638565063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,8,8,128,0,1,float16,float16,0,0.07645866771539052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,8,2,128,0,1,float16,float16,0,0.040448000033696495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,8,8,128,0,1,float16,float16,0,0.04249600072701772
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,8,4,128,0,1,float16,float16,0,0.040789333482583366
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,8,1,128,0,1,float16,float16,0,0.031061333914597828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,8,8,128,0,1,float16,float16,0,0.032255999743938446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,8,2,128,0,1,float16,float16,0,0.03089066594839096
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,8,8,128,0,1,float16,float16,0,0.023210667073726654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,8,4,128,0,1,float16,float16,0,0.11963733037312825
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,8,2,128,0,1,float16,float16,0,0.023039999107519787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,8,4,128,0,1,float16,float16,0,0.023381332556406658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,8,8,128,0,1,float16,float16,0,0.02184533327817917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,8,4,128,0,1,float16,float16,0,0.0315733328461647
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,8,1,128,0,1,float16,float16,0,0.021674667795499165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,8,2,128,0,1,float16,float16,0,0.021674667795499165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,8,1,128,0,1,float16,float16,0,0.023210667073726654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,8,1,128,0,1,float16,float16,0,0.1699840029080709
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,8,2,128,0,1,float16,float16,0,0.17510400215784708
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,8,4,128,0,1,float16,float16,0,0.02184533327817917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,128,8,1,128,0,1,float16,float16,0,0.3227306604385376
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,128,8,2,128,0,1,float16,float16,0,0.33348266283671063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,128,8,4,128,0,1,float16,float16,0,0.3653973340988159
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,8,4,128,0,1,float16,float16,0,0.19490132729212442
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,8,1,128,0,1,float16,float16,0,0.09147733449935913
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,8,8,128,0,1,float16,float16,0,0.24285866816838583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,8,8,128,0,1,float16,float16,0,0.13397333025932312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,8,4,128,0,1,float16,float16,0,0.13397333025932312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,8,8,128,0,1,float16,float16,0,0.06843733290831248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,8,1,128,0,1,float16,float16,0,0.049322664737701416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,8,1,128,0,1,float16,float16,0,0.03362133353948593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,8,4,128,0,1,float16,float16,0,0.053930665055910744
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,8,2,128,0,1,float16,float16,0,0.09659733374913533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,8,2,128,0,1,float16,float16,0,0.034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,8,2,128,0,1,float16,float16,0,0.050517335534095764
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,8,4,128,0,1,float16,float16,0,0.03549866626660029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,8,8,128,0,1,float16,float16,0,0.0266239990790685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,8,1,128,0,1,float16,float16,0,0.024746666351954143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,8,2,128,0,1,float16,float16,0,0.025087999800841015
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,8,8,128,0,1,float16,float16,0,0.018602666755517323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,8,4,128,0,1,float16,float16,0,0.01877333347996076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,8,8,128,0,1,float16,float16,0,0.0365226666132609
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,8,2,128,0,1,float16,float16,0,0.01877333347996076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,8,8,128,0,1,float16,float16,0,0.017237332959969837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,8,1,128,0,1,float16,float16,0,0.01877333347996076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,8,1,128,0,1,float16,float16,0,0.017237332959969837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,8,2,128,0,1,float16,float16,0,0.017237332959969837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,8,4,128,0,1,float16,float16,0,0.017407999684413273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,8,1,128,0,1,float16,float16,0,0.01672533278663953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,8,2,128,0,1,float16,float16,0,0.016554666062196095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,8,4,128,0,1,float16,float16,0,0.02611200014750163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,8,4,128,0,1,float16,float16,0,0.016554666062196095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,64,8,1,128,0,1,float16,float16,0,0.1713493267695109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,8,8,128,0,1,float16,float16,0,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,64,8,2,128,0,1,float16,float16,0,0.1771519978841146
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,8,1,128,0,1,float16,float16,0,0.09181867043177287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,8,8,128,0,1,float16,float16,0,0.13499733805656433
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,64,8,4,128,0,1,float16,float16,0,0.21094399690628052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,8,4,128,0,1,float16,float16,0,0.10973866780598958
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,8,8,128,0,1,float16,float16,0,0.06843733290831248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,8,2,128,0,1,float16,float16,0,0.05222400029500326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,8,8,128,0,1,float16,float16,0,0.03703466554482778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,8,1,128,0,1,float16,float16,0,0.03566933423280716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,8,4,128,0,1,float16,float16,0,0.05478399991989136
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,8,2,128,0,1,float16,float16,0,0.0363520011305809
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,8,2,128,0,1,float16,float16,0,0.09847467144330342
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,8,4,128,0,1,float16,float16,0,0.03703466554482778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,8,1,128,0,1,float16,float16,0,0.05034666756788889
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,8,8,128,0,1,float16,float16,0,0.025600001215934753
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,8,1,128,0,1,float16,float16,0,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,8,2,128,0,1,float16,float16,0,0.025087999800841015
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,8,4,128,0,1,float16,float16,0,0.025258667767047882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,8,8,128,0,1,float16,float16,0,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,8,2,128,0,1,float16,float16,0,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,8,8,128,0,1,float16,float16,0,0.016895999511082966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,8,1,128,0,1,float16,float16,0,0.017237332959969837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,8,2,128,0,1,float16,float16,0,0.017237332959969837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,8,4,128,0,1,float16,float16,0,0.0170666662355264
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,8,8,128,0,1,float16,float16,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,8,4,128,0,1,float16,float16,0,0.01877333347996076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,8,1,128,0,1,float16,float16,0,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,8,2,128,0,1,float16,float16,0,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,8,8,128,0,1,float16,float16,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,8,4,128,0,1,float16,float16,0,0.016554666062196095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,8,1,128,0,1,float16,float16,0,0.018432000031073887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,8,1,128,0,1,float16,float16,0,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,8,2,128,0,1,float16,float16,0,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,32,8,1,128,0,1,float16,float16,0,0.10018133123715718
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,32,8,2,128,0,1,float16,float16,0,0.10564266641934712
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,8,2,128,0,1,float16,float16,0,0.05273599922657013
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,8,8,128,0,1,float16,float16,0,0.07116800049940745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,32,8,4,128,0,1,float16,float16,0,0.11502933502197266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,8,4,128,0,1,float16,float16,0,0.05649066468079885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,8,4,128,0,1,float16,float16,0,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,8,8,128,0,1,float16,float16,0,0.03549866626660029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,8,1,128,0,1,float16,float16,0,0.0341333324710528
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,8,2,128,0,1,float16,float16,0,0.034645333886146545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,8,4,128,0,1,float16,float16,0,0.03549866626660029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,8,8,128,0,1,float16,float16,0,0.0266239990790685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,8,1,128,0,1,float16,float16,0,0.025941332181294758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,8,1,128,0,1,float16,float16,0,0.05120000243186951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,8,8,128,0,1,float16,float16,0,0.016895999511082966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,8,4,128,0,1,float16,float16,0,0.026965332527955372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,8,2,128,0,1,float16,float16,0,0.025941332181294758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,8,1,128,0,1,float16,float16,0,0.017407999684413273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,8,4,128,0,1,float16,float16,0,0.017237332959969837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,8,8,128,0,1,float16,float16,0,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,8,2,128,0,1,float16,float16,0,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,8,2,128,0,1,float16,float16,0,0.017407999684413273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,8,4,128,0,1,float16,float16,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,8,1,128,0,1,float16,float16,0,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,8,2,128,0,1,float16,float16,0,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,8,4,128,0,1,float16,float16,0,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,8,8,128,0,1,float16,float16,0,0.013823999712864557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,8,1,128,0,1,float16,float16,0,0.0145066666106383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,8,2,128,0,1,float16,float16,0,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,8,1,128,0,1,float16,float16,0,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,8,4,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,8,1,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,8,8,128,0,1,float16,float16,0,0.013653332988421122
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,8,8,128,0,1,float16,float16,0,0.013823999712864557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,8,2,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,8,4,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,16,8,1,128,0,1,float16,float16,0,0.07372800012429555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,8,1,128,0,1,float16,float16,0,0.04351999859015147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,16,8,4,128,0,1,float16,float16,0,0.07935999830563863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,8,8,128,0,1,float16,float16,0,0.04369066655635834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,8,4,128,0,1,float16,float16,0,0.04437333345413208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,8,1,128,0,1,float16,float16,0,0.030037333567937214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,8,2,128,0,1,float16,float16,0,0.03089066594839096
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,8,8,128,0,1,float16,float16,0,0.029866665601730347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,8,8,128,0,1,float16,float16,0,0.020821332931518555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,8,4,128,0,1,float16,float16,0,0.03054933249950409
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,16,8,2,128,0,1,float16,float16,0,0.07526400188604991
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,8,2,128,0,1,float16,float16,0,0.04386133452256521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,8,2,128,0,1,float16,float16,0,0.021162666380405426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,8,4,128,0,1,float16,float16,0,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,8,1,128,0,1,float16,float16,0,0.021162666380405426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,8,1,128,0,1,float16,float16,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,8,8,128,0,1,float16,float16,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,8,2,128,0,1,float16,float16,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,8,1,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,8,2,128,0,1,float16,float16,0,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,8,8,128,0,1,float16,float16,0,0.013653332988421122
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,8,2,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,8,4,128,0,1,float16,float16,0,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,8,4,128,0,1,float16,float16,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,8,1,128,0,1,float16,float16,0,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,8,8,128,0,1,float16,float16,0,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,8,4,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,8,8,128,0,1,float16,float16,0,0.013482666263977686
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,8,1,128,0,1,float16,float16,0,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,8,2,128,0,1,float16,float16,0,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,8,1,128,0,1,float16,float16,0,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,8,4,128,0,1,float16,float16,0,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,8,4,128,0,1,float16,float16,0,0.013823999712864557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,8,2,128,0,1,float16,float16,0,0.013653332988421122
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,8,8,128,0,1,float16,float16,0,0.01331199953953425
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,4,1,128,0,1,float16,float16,0,7.43936030069987
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,4,4,128,0,1,float16,float16,0,3.935061454772949
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,4,2,128,0,1,float16,float16,0,7.552853266398112
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,4,1,128,0,1,float16,float16,0,3.9548587799072266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16384,4,4,128,0,1,float16,float16,0,7.691946665445964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16384,4,1,128,0,1,float16,float16,0,15.082667032877604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16384,4,2,128,0,1,float16,float16,0,3.965440114339193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,4,1,128,0,1,float16,float16,0,2.1609813372294107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,4,4,128,0,1,float16,float16,0,2.173269271850586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16384,4,2,128,0,1,float16,float16,0,14.954325358072916
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16384,4,2,128,0,1,float16,float16,0,2.1580799420674643
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,4,1,128,0,1,float16,float16,0,2.305877367655436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,4,4,128,0,1,float16,float16,0,2.3284053802490234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,4,2,128,0,1,float16,float16,0,4.32093874613444
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,12288,4,2,128,0,1,float16,float16,0,2.3203840255737305
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,4,4,128,0,1,float16,float16,0,4.37333329518636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,4,1,128,0,1,float16,float16,0,1.3163519700368245
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,12288,4,2,128,0,1,float16,float16,0,8.614741643269857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,12288,4,1,128,0,1,float16,float16,0,4.348074595133464
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,4,4,128,0,1,float16,float16,0,1.3284693559010823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,12288,4,1,128,0,1,float16,float16,0,8.59170150756836
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,12288,4,2,128,0,1,float16,float16,0,1.3335893948872883
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,4,1,128,0,1,float16,float16,0,3.069610595703125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,4,1,128,0,1,float16,float16,0,1.6677546501159668
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,4,4,128,0,1,float16,float16,0,1.6974506378173828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,10240,4,2,128,0,1,float16,float16,0,1.6691199938456218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,10240,4,1,128,0,1,float16,float16,0,5.970432281494141
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,4,4,128,0,1,float16,float16,0,3.099818547566732
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,10240,4,2,128,0,1,float16,float16,0,6.094677607218425
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,4,4,128,0,1,float16,float16,0,0.9615360101064047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,4,2,128,0,1,float16,float16,0,0.9533440272013346
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,10240,4,1,128,0,1,float16,float16,0,0.9509546756744385
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,10240,4,2,128,0,1,float16,float16,0,3.1081813176472983
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,4,1,128,0,1,float16,float16,0,3.9069013595581055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,4,4,128,0,1,float16,float16,0,2.0626773834228516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,4,1,128,0,1,float16,float16,0,2.0341760317484536
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,8192,4,1,128,0,1,float16,float16,0,7.660544077555339
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,4,2,128,0,1,float16,float16,0,3.9475199381510415
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,8192,4,4,128,0,1,float16,float16,0,3.911168098449707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,4,1,128,0,1,float16,float16,0,1.1306666533152263
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,8192,4,2,128,0,1,float16,float16,0,2.056704044342041
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,4,4,128,0,1,float16,float16,0,1.1342506408691406
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,4,1,128,0,1,float16,float16,0,0.6604799826939901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,8192,4,2,128,0,1,float16,float16,0,1.12827730178833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,8192,4,2,128,0,1,float16,float16,0,7.713450749715169
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,4,4,128,0,1,float16,float16,0,0.6626986662546793
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,8192,4,2,128,0,1,float16,float16,0,0.6635520060857137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,4,1,128,0,1,float16,float16,0,1.2125866413116455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,4,1,128,0,1,float16,float16,0,2.271402676900228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,4,4,128,0,1,float16,float16,0,1.2434773445129395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,6144,4,2,128,0,1,float16,float16,0,1.2144640286763508
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,6144,4,1,128,0,1,float16,float16,0,4.425898551940918
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,4,4,128,0,1,float16,float16,0,2.307583967844645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,4,4,128,0,1,float16,float16,0,0.6930773258209229
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,4,1,128,0,1,float16,float16,0,0.6912000179290771
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,6144,4,2,128,0,1,float16,float16,0,2.284202734629313
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,4,4,128,0,1,float16,float16,0,0.4360533157984416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,6144,4,2,128,0,1,float16,float16,0,0.6930773258209229
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,4,1,128,0,1,float16,float16,0,0.4278613328933716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,6144,4,2,128,0,1,float16,float16,0,0.4370773235956828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,6144,4,2,128,0,1,float16,float16,0,4.493141492207845
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,4,1,128,0,1,float16,float16,0,1.0939733187357585
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,4,4,128,0,1,float16,float16,0,1.121280034383138
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,4,1,128,0,1,float16,float16,0,2.1055146853129068
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,4,2,128,0,1,float16,float16,0,2.1104639371236167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,4096,4,1,128,0,1,float16,float16,0,4.09497610727946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,4096,4,2,128,0,1,float16,float16,0,1.1115520000457764
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,4,4,128,0,1,float16,float16,0,0.6155946652094523
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,4,1,128,0,1,float16,float16,0,0.6000639994939169
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,4096,4,4,128,0,1,float16,float16,0,2.132138729095459
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,4096,4,2,128,0,1,float16,float16,0,4.182016054789226
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,4096,4,2,128,0,1,float16,float16,0,0.6111573378245035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,4,1,128,0,1,float16,float16,0,0.3577173153559367
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,4,4,128,0,1,float16,float16,0,0.36351998647054035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,4,4,128,0,1,float16,float16,0,0.233130673567454
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,4,1,128,0,1,float16,float16,0,0.23176532983779907
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,4096,4,2,128,0,1,float16,float16,0,0.23398399353027344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,4096,4,2,128,0,1,float16,float16,0,0.3628373146057129
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,4,1,128,0,1,float16,float16,0,0.659114678700765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,4,4,128,0,1,float16,float16,0,0.6852266788482666
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,4,1,128,0,1,float16,float16,0,1.2622506618499756
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,4,2,128,0,1,float16,float16,0,1.2695893446604412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,3072,4,1,128,0,1,float16,float16,0,2.4321707089742026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,3072,4,4,128,0,1,float16,float16,0,1.2880213260650635
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,3072,4,2,128,0,1,float16,float16,0,0.6908586819966634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,4,1,128,0,1,float16,float16,0,0.3763200044631958
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,3072,4,2,128,0,1,float16,float16,0,2.468693256378174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,4,4,128,0,1,float16,float16,0,0.38417065143585205
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,4,1,128,0,1,float16,float16,0,0.2259626587231954
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,4,4,128,0,1,float16,float16,0,0.2379093368848165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,3072,4,2,128,0,1,float16,float16,0,0.3805866638819377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,3072,4,2,128,0,1,float16,float16,0,0.2336426575978597
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,4,1,128,0,1,float16,float16,0,0.15411200126012167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,4,2,128,0,1,float16,float16,0,0.1539413332939148
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,3072,4,4,128,0,1,float16,float16,0,0.1546239952246348
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,4,1,128,0,1,float16,float16,0,0.6316373348236084
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,4,1,128,0,1,float16,float16,0,1.2139519850413005
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,4,2,128,0,1,float16,float16,0,1.2310187021891277
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,4,4,128,0,1,float16,float16,0,0.6640640099843343
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,2048,4,2,128,0,1,float16,float16,0,0.6439253489176432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,2048,4,4,128,0,1,float16,float16,0,1.2651519775390625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,4,4,128,0,1,float16,float16,0,0.3568640152613322
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,4,4,128,0,1,float16,float16,0,0.20497065782546997
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,4,1,128,0,1,float16,float16,0,0.33723731835683185
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,2048,4,2,128,0,1,float16,float16,0,0.34679468472798664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,2048,4,2,128,0,1,float16,float16,0,2.4130560557047525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,2048,4,1,128,0,1,float16,float16,0,2.3673173586527505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,4,2,128,0,1,float16,float16,0,0.20138667027155557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,4,4,128,0,1,float16,float16,0,0.12868266304334006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,2048,4,1,128,0,1,float16,float16,0,0.1971199909845988
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,4,1,128,0,1,float16,float16,0,0.1281706690788269
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,4,4,128,0,1,float16,float16,0,0.1032533347606659
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,4,1,128,0,1,float16,float16,0,0.10291199882825215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,2048,4,2,128,0,1,float16,float16,0,0.12731732924779257
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,4,1,128,0,1,float16,float16,0,0.3983360131581624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,4,1,128,0,1,float16,float16,0,0.7574186325073242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,2048,4,2,128,0,1,float16,float16,0,0.10291199882825215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,4,4,128,0,1,float16,float16,0,0.42052265008290607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,4,2,128,0,1,float16,float16,0,0.7720959981282552
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1536,4,1,128,0,1,float16,float16,0,1.4619305928548176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1536,4,4,128,0,1,float16,float16,0,0.7978666623433431
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1536,4,2,128,0,1,float16,float16,0,1.498794714609782
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,4,4,128,0,1,float16,float16,0,0.23296000560124716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,4,2,128,0,1,float16,float16,0,0.22425599892934164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,4,4,128,0,1,float16,float16,0,0.14079999923706055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1536,4,2,128,0,1,float16,float16,0,0.4049919843673706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,4,1,128,0,1,float16,float16,0,0.1358506679534912
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,4,4,128,0,1,float16,float16,0,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,4,1,128,0,1,float16,float16,0,0.08089600006739299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,4,4,128,0,1,float16,float16,0,0.06877866884072621
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1536,4,2,128,0,1,float16,float16,0,0.08055466910203297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,4,2,128,0,1,float16,float16,0,0.06690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1536,4,1,128,0,1,float16,float16,0,0.21606399615605673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1536,4,2,128,0,1,float16,float16,0,0.13482667009035745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1536,4,1,128,0,1,float16,float16,0,0.06690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,4,1,128,0,1,float16,float16,0,0.7098026275634766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,4,1,128,0,1,float16,float16,0,0.3758080005645752
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,4,2,128,0,1,float16,float16,0,0.7316479682922363
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,4,4,128,0,1,float16,float16,0,0.4148906469345093
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,1024,4,1,128,0,1,float16,float16,0,1.3870080312093098
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,1024,4,4,128,0,1,float16,float16,0,0.7922346591949463
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,4,4,128,0,1,float16,float16,0,0.22698666652043661
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,4,2,128,0,1,float16,float16,0,0.27033599217732746
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,1024,4,1,128,0,1,float16,float16,0,0.20616533358891806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,1024,4,2,128,0,1,float16,float16,0,1.4281387329101562
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,4,1,128,0,1,float16,float16,0,0.12032000223795573
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,1024,4,2,128,0,1,float16,float16,0,0.3843413194020589
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,4,2,128,0,1,float16,float16,0,0.1220266620318095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,1024,4,4,128,0,1,float16,float16,0,0.12902399897575378
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,4,4,128,0,1,float16,float16,0,0.07901866734027863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,4,1,128,0,1,float16,float16,0,0.07458133498827617
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,4,4,128,0,1,float16,float16,0,0.06092800199985504
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,4,1,128,0,1,float16,float16,0,0.05922133227189382
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,4,4,128,0,1,float16,float16,0,0.04659200211366018
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,1024,4,2,128,0,1,float16,float16,0,0.07509333391984303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,4,2,128,0,1,float16,float16,0,0.04625066618124644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,1024,4,1,128,0,1,float16,float16,0,0.04607999821503957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,1024,4,2,128,0,1,float16,float16,0,0.05922133227189382
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,4,1,128,0,1,float16,float16,0,0.4884479840596517
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,4,4,128,0,1,float16,float16,0,0.2995199958483378
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,512,4,1,128,0,1,float16,float16,0,0.9541973272959391
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,4,4,128,0,1,float16,float16,0,0.5768533150355021
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,512,4,2,128,0,1,float16,float16,0,0.5118293364842733
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,4,4,128,0,1,float16,float16,0,0.1634986698627472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,4,1,128,0,1,float16,float16,0,0.14335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,4,1,128,0,1,float16,float16,0,0.25804799795150757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,512,4,2,128,0,1,float16,float16,0,0.9992533524831136
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,512,4,2,128,0,1,float16,float16,0,0.2693120042483012
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,512,4,2,128,0,1,float16,float16,0,0.15018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,4,4,128,0,1,float16,float16,0,0.09233066439628601
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,4,2,128,0,1,float16,float16,0,0.08345599969228108
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,4,4,128,0,1,float16,float16,0,0.05495466788609823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,4,2,128,0,1,float16,float16,0,0.05341866612434387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,512,4,1,128,0,1,float16,float16,0,0.08004266520341237
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,4,1,128,0,1,float16,float16,0,0.03874133278926214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,4,2,128,0,1,float16,float16,0,0.03942399968703588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,4,1,128,0,1,float16,float16,0,0.031061333914597828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,512,4,1,128,0,1,float16,float16,0,0.05222400029500326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,4,4,128,0,1,float16,float16,0,0.03089066594839096
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,512,4,2,128,0,1,float16,float16,0,0.03089066594839096
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,4,1,128,0,1,float16,float16,0,0.029525332152843475
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,4,2,128,0,1,float16,float16,0,0.029696000119050343
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,512,4,4,128,0,1,float16,float16,0,0.02935466667016347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,512,4,4,128,0,1,float16,float16,0,0.040106666584809623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,4,1,128,0,1,float16,float16,0,0.22340265909830728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,256,4,1,128,0,1,float16,float16,0,0.3809279998143514
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,4,4,128,0,1,float16,float16,0,0.2563413381576538
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,4,4,128,0,1,float16,float16,0,0.1423360009988149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,256,4,2,128,0,1,float16,float16,0,0.21282132466634116
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,4,1,128,0,1,float16,float16,0,0.11161599556605022
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,4,4,128,0,1,float16,float16,0,0.07441066702206929
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,256,4,2,128,0,1,float16,float16,0,0.1181013286113739
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,4,1,128,0,1,float16,float16,0,0.06126933296521505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,4,1,128,0,1,float16,float16,0,0.040618665516376495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,4,4,128,0,1,float16,float16,0,0.04232533276081085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,256,4,2,128,0,1,float16,float16,0,0.4041386842727661
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,256,4,2,128,0,1,float16,float16,0,0.06621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,4,4,128,0,1,float16,float16,0,0.023381332556406658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,4,2,128,0,1,float16,float16,0,0.031061333914597828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,4,4,128,0,1,float16,float16,0,0.02184533327817917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,256,4,2,128,0,1,float16,float16,0,0.040789333482583366
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,4,1,128,0,1,float16,float16,0,0.023210667073726654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,256,4,2,128,0,1,float16,float16,0,0.023210667073726654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,4,1,128,0,1,float16,float16,0,0.030378667016824085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,4,1,128,0,1,float16,float16,0,0.02184533327817917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,256,4,2,128,0,1,float16,float16,0,0.021674667795499165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,4,4,128,0,1,float16,float16,0,0.02065066620707512
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,256,4,4,128,0,1,float16,float16,0,0.032255999743938446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,4,2,128,0,1,float16,float16,0,0.021333334346612293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,256,4,1,128,0,1,float16,float16,0,0.020992000897725422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,4,1,128,0,1,float16,float16,0,0.09506133198738098
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,4,4,128,0,1,float16,float16,0,0.13158399860064188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,4,1,128,0,1,float16,float16,0,0.05017599960168203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,128,4,1,128,0,1,float16,float16,0,0.17220266660054526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,128,4,2,128,0,1,float16,float16,0,0.10478933652242024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,4,1,128,0,1,float16,float16,0,0.034645333886146545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,4,4,128,0,1,float16,float16,0,0.03703466554482778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,4,4,128,0,1,float16,float16,0,0.06587733328342438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,128,4,2,128,0,1,float16,float16,0,0.19353600343068442
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,4,1,128,0,1,float16,float16,0,0.025429333249727886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,128,4,2,128,0,1,float16,float16,0,0.03549866626660029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,4,4,128,0,1,float16,float16,0,0.0264533335963885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,128,4,2,128,0,1,float16,float16,0,0.025429333249727886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,4,4,128,0,1,float16,float16,0,0.018602666755517323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,128,4,2,128,0,1,float16,float16,0,0.053077335158983864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,4,4,128,0,1,float16,float16,0,0.017407999684413273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,4,1,128,0,1,float16,float16,0,0.017407999684413273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,4,4,128,0,1,float16,float16,0,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,4,1,128,0,1,float16,float16,0,0.016554666062196095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,4,4,128,0,1,float16,float16,0,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,4,1,128,0,1,float16,float16,0,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,128,4,2,128,0,1,float16,float16,0,0.016554666062196095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,4,2,128,0,1,float16,float16,0,0.01877333347996076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,128,4,1,128,0,1,float16,float16,0,0.01877333347996076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,128,4,2,128,0,1,float16,float16,0,0.016554666062196095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,128,4,2,128,0,1,float16,float16,0,0.017237332959969837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,64,4,1,128,0,1,float16,float16,0,0.09727999567985535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,4,4,128,0,1,float16,float16,0,0.06741333504517873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,4,1,128,0,1,float16,float16,0,0.050517335534095764
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,64,4,2,128,0,1,float16,float16,0,0.10871466994285583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,4,4,128,0,1,float16,float16,0,0.025258667767047882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,4,1,128,0,1,float16,float16,0,0.03549866626660029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,64,4,2,128,0,1,float16,float16,0,0.053930665055910744
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,4,1,128,0,1,float16,float16,0,0.024746666351954143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,64,4,2,128,0,1,float16,float16,0,0.025258667767047882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,4,2,128,0,1,float16,float16,0,0.03618133316437403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,4,4,128,0,1,float16,float16,0,0.018602666755517323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,4,1,128,0,1,float16,float16,0,0.018432000031073887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,64,4,4,128,0,1,float16,float16,0,0.03737599899371465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,4,2,128,0,1,float16,float16,0,0.017237332959969837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,4,1,128,0,1,float16,float16,0,0.017237332959969837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,4,1,128,0,1,float16,float16,0,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,64,4,2,128,0,1,float16,float16,0,0.01877333347996076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,4,2,128,0,1,float16,float16,0,0.016554666062196095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,64,4,4,128,0,1,float16,float16,0,0.01672533278663953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,64,4,4,128,0,1,float16,float16,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,4,2,128,0,1,float16,float16,0,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,4,1,128,0,1,float16,float16,0,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,4,4,128,0,1,float16,float16,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,64,4,4,128,0,1,float16,float16,0,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,4,2,128,0,1,float16,float16,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,4,1,128,0,1,float16,float16,0,0.034815999368826546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,64,4,1,128,0,1,float16,float16,0,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,4,4,128,0,1,float16,float16,0,0.0365226666132609
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,32,4,2,128,0,1,float16,float16,0,0.03566933423280716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,4,1,128,0,1,float16,float16,0,0.02611200014750163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,32,4,1,128,0,1,float16,float16,0,0.053247998158137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,4,4,128,0,1,float16,float16,0,0.0264533335963885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,32,4,2,128,0,1,float16,float16,0,0.02628266563018163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,4,1,128,0,1,float16,float16,0,0.01757866640885671
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,4,4,128,0,1,float16,float16,0,0.014848000059525171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,4,2,128,0,1,float16,float16,0,0.01757866640885671
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,32,4,2,128,0,1,float16,float16,0,0.05717333157857259
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,4,1,128,0,1,float16,float16,0,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,4,4,128,0,1,float16,float16,0,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,4,1,128,0,1,float16,float16,0,0.014677333335081736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,32,4,2,128,0,1,float16,float16,0,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,32,4,4,128,0,1,float16,float16,0,0.017237332959969837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,4,4,128,0,1,float16,float16,0,0.013823999712864557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,32,4,2,128,0,1,float16,float16,0,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,4,2,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,4,4,128,0,1,float16,float16,0,0.013653332988421122
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,4,1,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,32,4,2,128,0,1,float16,float16,0,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,4,1,128,0,1,float16,float16,0,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,32,4,1,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,4,4,128,0,1,float16,float16,0,0.013482666263977686
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,4,1,128,0,1,float16,float16,0,0.030720000465710957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,16,4,2,128,0,1,float16,float16,0,0.045909335215886436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,4,2,128,0,1,float16,float16,0,0.031231999397277832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,32,4,2,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,128,16,4,4,128,0,1,float16,float16,0,0.030720000465710957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,256,16,4,1,128,0,1,float16,float16,0,0.04471466441949209
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,4,4,128,0,1,float16,float16,0,0.021162666380405426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,4,2,128,0,1,float16,float16,0,0.021674667795499165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,4,4,128,0,1,float16,float16,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,4,1,128,0,1,float16,float16,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,4,4,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,4,1,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,64,16,4,1,128,0,1,float16,float16,0,0.021674667795499165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,32,16,4,2,128,0,1,float16,float16,0,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,4,1,128,0,1,float16,float16,0,0.013823999712864557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,4,4,128,0,1,float16,float16,0,0.013653332988421122
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,8,16,4,2,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,4,4,128,0,1,float16,float16,0,0.013653332988421122
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,16,16,4,2,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,4,1,128,0,1,float16,float16,0,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,4,1,128,0,1,float16,float16,0,0.013823999712864557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,4,2,128,0,1,float16,float16,0,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,4,4,128,0,1,float16,float16,0,0.01331199953953425
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,4,1,128,0,1,float16,float16,0,0.013994666437307993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,2,16,4,4,128,0,1,float16,float16,0,0.013482666263977686
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,4,16,4,2,128,0,1,float16,float16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,context_attention,torch_flow,1,16,4,2,128,0,1,float16,float16,0,0.013823999712864557
