framework,version,device,op_name,kernel_source,batch_size,isl,num_heads,num_key_value_heads,head_dim,window_size,beam_width,attn_dtype,kv_cache_dtype,step,latency
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,1,0.0170666662355264
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,3,0.04898133377234141
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,1,0.04898133377234141
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,7,0.051029334465662636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,3,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,15,0.01791999985774358
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,15,0.05444266895453135
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,31,0.022357332209746044
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,63,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,255,0.030378667016824085
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,7,0.017407999684413273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,63,0.0718506673971812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,511,0.19541333119074503
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,127,0.023381332556406658
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,511,0.16452266772588095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,255,0.09096533060073853
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,127,0.0718506673971812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,31,0.07167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,1023,0.3863893349965413
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,1023,0.3290453354517619
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,2047,0.6418773333231608
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,2047,1.3412693341573079
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,1,0.007397333160042763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,4095,2.8874985376993814
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,4095,1.345194657643636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,1,0.011296000331640244
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,3,0.011440000186363855
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,15,0.007381333038210869
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,15,0.011114666859308878
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,31,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,31,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,63,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,255,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,127,0.013781332721312841
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,255,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,511,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,511,0.018090666582187016
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,1023,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,1023,0.03259733319282532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,2047,0.022154666483402252
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,2047,0.048810665806134544
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,4095,0.03289066751797994
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,4095,0.07611733178297679
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,8191,0.0481279989083608
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,8191,0.13636266191800436
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,16383,0.25036799907684326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,3,0.007477333148320516
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,16383,0.2167466680208842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,15,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,31,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,31,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,63,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,63,0.008373333141207695
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,127,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,255,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,511,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,511,0.0315733328461647
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,1023,0.021503999829292297
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,255,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,2047,0.02735466758410136
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,1023,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,2047,0.06572799881299336
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,4095,0.04281599819660187
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,4095,0.10905599594116211
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,8191,0.2206666668256124
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,8191,0.19643733898798624
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,16383,0.6985387007395426
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,16383,0.39185067017873126
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,1,0.03089066594839096
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,1,0.0955573320388794
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,3,0.03105599929889043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,3,0.09540266791979472
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,7,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,7,0.10001066327095032
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,15,0.0328053335348765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,15,0.10359467069307964
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,31,0.040448000033696495
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,31,0.13431466619173685
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,63,0.04113066693147024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,63,0.13448533415794373
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,127,0.04249600072701772
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,127,0.13499733805656433
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,255,0.17117865880330405
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,255,0.19350399573644003
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,511,0.32546132802963257
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,1023,0.7487146854400635
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,511,0.3778560161590576
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,1023,0.6138879855473837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,2047,1.1979093551635742
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,2047,2.8503039677937827
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,1,0.006826666494210561
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,1,0.020303999384244282
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,3,0.006826666494210561
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,3,0.020469332734743755
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,7,0.020138667275508244
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,7,0.007333333293596904
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,15,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,15,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,31,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,63,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,31,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,127,0.009189333145817121
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,63,0.025258667767047882
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,255,0.008714666590094566
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,127,0.025770666698614757
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,255,0.025770666698614757
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,511,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,511,0.031744000812371574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,1023,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,1023,0.05273599922657013
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,2047,0.02918400118748347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,2047,0.09540266791979472
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,4095,0.18107734123865762
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,4095,0.3619840145111084
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,8191,0.7296000321706136
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,8191,0.37546666463216144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,1,0.05751466751098633
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,16383,0.7787520090738932
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,16383,1.4361599286397297
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,1,0.1867093245188395
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,3,0.05734399954477946
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,3,0.18722132841746011
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,7,0.059562668204307556
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,7,0.19710399707158408
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,15,0.06195199986298879
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,15,0.20548266172409058
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,31,0.2630186676979065
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,31,0.07765333354473114
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,63,0.07863999903202057
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,127,0.19490132729212442
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,63,0.26368000109990436
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,127,0.26385066906611127
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,255,0.34707732995351154
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,255,0.3781973520914714
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,511,0.7435946464538574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,511,0.6357333262761434
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,1023,1.4750720659891765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,1023,1.1999573707580566
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,float16,1,0.11059199770291646
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,fp8,1,0.36522666613260907
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,float16,3,0.1104213297367096
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,fp8,3,0.3657386700312297
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,float16,7,0.11468799908955891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,float16,15,0.11986133456230164
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,fp8,7,0.38709867000579834
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,fp8,15,0.4037439823150635
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,fp8,31,0.5195093154907227
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,float16,31,0.15187199910481772
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,float16,63,0.2027519941329956
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,fp8,63,0.5208746592203776
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,float16,127,0.38229334354400635
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,fp8,127,0.5340160131454468
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,float16,255,0.7480426629384359
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,fp8,255,0.6830080350240072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,float16,1,0.21606399615605673
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,float16,3,0.21639466285705566
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,fp8,3,0.7270399729410807
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,fp8,1,0.72652800877889
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,float16,7,0.22494399547576904
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,fp8,7,0.7703573703765869
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,float16,15,0.2409813404083252
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,float16,31,0.3193173408508301
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,fp8,15,0.803669293721517
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,fp8,31,1.0315093199412029
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,float16,63,0.3968000014623006
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,fp8,63,1.0545492966969807
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,float16,127,0.7572480042775472
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,fp8,127,1.0579626560211182
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,1,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,1,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,3,0.0100853331387043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,3,0.02029866725206375
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,7,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,15,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,7,0.021162666380405426
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,15,0.021333334346612293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,31,0.023552000522613525
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,31,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,63,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,63,0.023381332556406658
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,127,0.013999999811251959
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,127,0.023728000621000927
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,255,0.01463466634353002
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,255,0.02935466667016347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,511,0.018432000031073887
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,1023,0.0314026673634847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,511,0.05120000243186951
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,1023,0.09471999605496724
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,2047,0.2039466698964437
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,2047,0.18244266510009766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,4095,0.6973439852396647
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,4095,0.38145601749420166
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,8191,1.4513492584228516
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,8191,0.7838719685872396
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,16383,2.8648106257120767
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,16383,2.2654293378194175
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,float16,1,0.4275199969609578
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,fp8,1,1.445034662882487
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,float16,3,0.42954667409261066
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,fp8,3,1.4458880424499512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,float16,7,0.47086934248606366
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,fp8,7,1.5440212885538738
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,float16,15,0.5203573306401571
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,fp8,15,1.6290133794148762
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,float16,31,0.6468266646067301
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,fp8,31,2.089301268259684
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,float16,63,0.7846826712290446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,fp8,63,2.0969813664754233
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,float16,1,0.9041919708251953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,fp8,1,2.891946792602539
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,float16,7,0.9854293664296468
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,float16,3,0.9478826522827148
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,fp8,3,2.9448534647623696
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,fp8,7,3.2093868255615234
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,float16,15,1.0371413230895996
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,fp8,15,3.3155412673950195
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,1,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,1,0.029850666721661884
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,float16,31,1.2595199743906658
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,3,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,3,0.029696000119050343
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,fp8,31,4.169386545817058
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,7,0.031061333914597828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,7,0.010106666634480158
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,15,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,31,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,15,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,31,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,63,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,63,0.04031466692686081
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,127,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,255,0.017237332959969837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,127,0.040448000033696495
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,255,0.05085866649945577
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,511,0.09062400460243225
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,511,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,1023,0.20087466637293497
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,1023,0.17249600092569986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,2047,0.701749324798584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,2047,0.3582293192545573
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,4095,1.3547520637512207
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,4095,0.7261866728464762
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,8191,1.580885410308838
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,8191,2.864591916402181
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,1,0.01738133281469345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,1,0.05222400029500326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,3,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,3,0.053930665055910744
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,7,0.056832000613212585
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,7,0.01844266677896182
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,31,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,15,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,15,0.07402666906515758
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,31,0.07389866809050243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,63,0.02350933353106181
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,63,0.07423999905586243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,127,0.030720000465710957
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,127,0.09281599521636963
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,255,0.19660800695419312
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,255,0.16671999295552573
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,511,0.38860801855723065
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,511,0.3331413269042969
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,1023,1.3458773295084636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,1023,0.6466559966405233
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,2047,2.8670291900634766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,2047,1.341610590616862
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,1,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,4095,2.9812053044637046
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,3,0.009765333185593287
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,4095,5.783722559611003
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,3,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,7,0.007173333317041397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,15,0.007626666376988093
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,7,0.013317332913478216
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,15,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,31,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,31,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,63,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,127,0.01482133318980535
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,255,0.008746666833758354
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,255,0.019626667102177937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,511,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,1023,0.022357332209746044
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,511,0.03498666733503342
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,1023,0.05120000243186951
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,2047,0.031744000812371574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,2047,0.07850666840871175
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,4095,0.04659200211366018
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,4095,0.14045866330464682
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,8191,0.21708800395329794
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,8191,0.2568533420562744
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,16383,0.6604799826939901
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,16383,0.5184853474299113
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,1,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,3,0.013493333011865616
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,3,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,7,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,7,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,15,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,15,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,31,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,31,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,63,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,127,0.008858666444818178
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,255,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,255,0.03379199902216593
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,511,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,511,0.04594666759173075
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,1023,0.02664000044266383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,1023,0.06797333558400472
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,2047,0.040789333482583366
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,2047,0.1129813293615977
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,4095,0.21708800395329794
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,4095,0.20241065820058188
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,8191,0.7096266746520996
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,8191,0.39882131417592365
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,16383,1.4561279614766438
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,16383,0.8268799781799316
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,1,0.031744000812371574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,1,0.10001066327095032
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,3,0.03310933212439219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,3,0.10393599669138591
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,7,0.03379199902216593
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,7,0.10769066214561462
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,15,0.04147200038035711
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,15,0.13823999961217245
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,31,0.13823999961217245
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,31,0.04164266586303711
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,63,0.043007999658584595
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,63,0.13858133554458618
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,127,0.19746132691701254
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,127,0.17390932639439902
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,255,0.38180267810821533
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,255,0.3313973347345988
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,511,0.7533226807912191
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,511,0.6208853324254354
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,1023,2.8583094278971353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,1023,1.2067840099334717
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,2047,2.512725353240967
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,1,0.022821334501107533
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,2047,5.7043412526448565
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,3,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,7,0.023210667073726654
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,15,0.007994666695594788
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,15,0.02422400067249934
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,63,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,63,0.027136000494162243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,31,0.026965332527955372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,31,0.00874133345981439
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,127,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,127,0.027477333943049114
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,255,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,255,0.03293866664171219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,511,0.0170666662355264
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,1023,0.028858666618665058
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,511,0.05461333195368449
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,1023,0.09779199957847595
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,2047,0.3619840145111084
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,2047,0.18397865692774454
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,4095,0.7195573647816976
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,4095,0.3800746599833171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,8191,1.4551199277242024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,8191,0.7906986872355143
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,16383,2.868570645650228
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,1,0.058880001306533813
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,1,0.19438934326171875
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,16383,1.538885275522868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,3,0.06109866499900818
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,3,0.20325867335001627
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,7,0.063509335120519
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,7,0.211626668771108
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,15,0.07970133423805237
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,15,0.27033599217732746
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,31,0.08055466910203297
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,31,0.2701066732406616
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,63,0.20070399840672812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,63,0.27050666014353436
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,127,0.38465599219004315
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,127,0.35601067543029785
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,255,0.644266684850057
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,255,0.7514293193817139
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,511,1.4829227129618328
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,511,1.2113920052846272
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,1023,2.362826665242513
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,1,0.1129813293615977
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,fp8,1,0.37717334429423016
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,1023,5.686442693074544
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,3,0.11841066678365071
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,fp8,3,0.3973120053609212
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,7,0.1237333317597707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,15,0.15769599874814352
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,fp8,7,0.41523198286692303
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,fp8,15,0.5328213373819987
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,31,0.21964800357818604
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,fp8,31,0.5343786478042603
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,63,0.3944266637166341
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,fp8,63,0.5512533187866211
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,127,0.7601493199666342
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,fp8,127,0.6976853211720785
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,fp8,255,1.2685653368632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,float16,1,0.2208426594734192
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,255,1.4875307083129883
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,fp8,1,0.7482026418050131
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,float16,3,0.233130673567454
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,float16,7,0.25600000222524005
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,fp8,3,0.7893173694610596
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,fp8,7,0.8253653049468994
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,float16,15,0.33058132727940875
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,fp8,15,1.0692266623179119
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,float16,31,0.4347093502680461
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,fp8,31,1.0806612968444824
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,float16,63,0.7840426762898763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,fp8,63,1.0885120232899983
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,float16,127,1.5117066701253254
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,1,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,1,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,fp8,127,1.3841066360473633
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,3,0.022986667851607006
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,3,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,7,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,7,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,15,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,15,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,31,0.013658666362365087
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,31,0.025061334172884624
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,63,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,127,0.014869333555301031
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,63,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,127,0.03054933249950409
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,255,0.0184906671444575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,255,0.05205333232879639
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,511,0.03173866619666418
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,511,0.09540266791979472
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,1023,0.20292266209920248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,1023,0.18397865692774454
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,2047,0.6975146929423014
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,2047,0.38230399290720624
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,4095,0.9069226582845052
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,4095,1.4349652926127117
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,8191,1.5643893877665203
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,8191,2.9076480865478516
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,16383,5.730159759521484
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,16383,4.511253356933594
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,float16,1,0.4648960034052531
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,fp8,1,1.495408058166504
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,float16,3,0.5075360139211019
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,fp8,3,1.614847977956136
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,float16,7,0.5376000006993612
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,fp8,7,1.7059839566548665
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,float16,15,0.6558719873428345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,fp8,15,2.149717330932617
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,float16,31,0.8635733127593994
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,fp8,31,2.1522773106892905
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,float16,63,1.5592106183369954
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,fp8,63,2.165034612019857
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,float16,1,0.9787733554840088
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,fp8,1,3.106303850809733
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,float16,3,1.0147840181986492
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,fp8,3,3.2802133560180664
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,float16,7,1.0699093341827393
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,float16,15,1.3071359793345134
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,fp8,7,3.410090764363607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,fp8,15,4.292096138000488
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,float16,31,1.71997865041097
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,1,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,1,0.03257599969704946
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,3,0.010426666587591171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,3,0.0334346666932106
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,fp8,31,4.293290774027507
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,7,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,15,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,7,0.034815999368826546
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,15,0.04164266586303711
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,31,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,63,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,31,0.04164266586303711
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,63,0.04164266586303711
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,127,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,127,0.05205333232879639
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,255,0.09198932846387227
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,255,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,511,0.20172800620396933
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,511,0.17390932639439902
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,1023,0.35942399501800537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,1023,0.7016106446584066
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,2047,1.4360106786092122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,2047,0.7292959690093994
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,4095,1.56876802444458
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,4095,2.8991146087646484
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,1,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,1,0.03942933430274328
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,3,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,8191,3.167914708455404
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,8191,5.7280534108479815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,3,0.03925333420435587
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,7,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,7,0.041189332803090416
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,15,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,31,0.017237332959969837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,15,0.042133331298828125
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,31,0.054272000988324486
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,63,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,63,0.053930665055910744
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,127,0.018245333184798557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,127,0.05444266895453135
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,255,0.023381332556406658
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,255,0.06843733290831248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,511,0.12270933389663696
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,511,0.045909335215886436
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,1023,0.29149866104125977
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,1023,0.2333013415336609
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,2047,0.483791987101237
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,2047,0.9883306821187338
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,4095,2.160639921824137
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,4095,1.006277322769165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,3,0.011120000233252844
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,7,0.007216000308593114
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,7,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,15,0.00730666642387708
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,15,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,31,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,63,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,31,0.01191466674208641
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,63,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,127,0.008736000085870424
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,255,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,255,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,511,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,511,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,1023,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,1023,0.032255999743938446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,2047,0.02102400114138921
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,2047,0.04852266609668732
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,4095,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,4095,0.07543466488520305
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,8191,0.04454400142033895
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,8191,0.1360213359196981
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,16383,0.07404266794522603
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,16383,0.24951465924580893
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,1,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,3,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,7,0.011946666985750198
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,15,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,31,0.007631999750932057
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,15,0.007333333293596904
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,31,0.012234666695197424
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,63,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,63,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,127,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,127,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,255,0.01369599997997284
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,255,0.009599999835093817
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,511,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,511,0.031231999397277832
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,1023,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,1023,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,2047,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,2047,0.0653653343518575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,4095,0.03858133405447006
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,4095,0.10854400197664897
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,8191,0.06565866867701213
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,8191,0.19632534186045328
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,16383,0.505514661471049
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,16383,0.37649067242940265
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,1,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,1,0.07543466488520305
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,3,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,3,0.07526400188604991
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,7,0.0791786660750707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,7,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,15,0.025600001215934753
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,15,0.08243200182914734
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,31,0.0314026673634847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,31,0.10307733217875163
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,63,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,63,0.10322133700052898
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,127,0.03290133426586787
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,127,0.10342400272687276
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,255,0.04625066618124644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,255,0.13090133666992188
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,511,0.28569600979487103
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,511,0.23995733261108398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,1023,0.5637120008468628
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,1023,0.47172268231709796
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,2047,0.9200853506724039
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,2047,2.0843520164489746
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,1,0.006826666494210561
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,1,0.020474666108687718
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,3,0.006826666494210561
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,3,0.019797333826621372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,7,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,7,0.019850666324297588
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,15,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,31,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,15,0.007189333438873291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,31,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,63,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,63,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,127,0.025216000775496166
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,255,0.025429333249727886
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,255,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,511,0.0314026673634847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,1023,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,511,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,1023,0.05216533442338308
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,2047,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,2047,0.09506133198738098
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,4095,0.04454400142033895
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,4095,0.18158932526906332
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,8191,0.544597347577413
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,8191,0.3566933472951253
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,16383,0.9997866948445638
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,1,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,16383,0.7649280230204264
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,1,0.1414346694946289
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,3,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,3,0.14148267110188803
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,7,0.045738667249679565
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,7,0.14847999811172485
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,15,0.04744533201058706
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,15,0.1551359991232554
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,31,0.059562668204307556
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,31,0.19985065857569376
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,63,0.0602453351020813
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,63,0.20053333044052124
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,127,0.08038400113582611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,127,0.20106667280197144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,255,0.2604373296101888
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,255,0.28547199567159015
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,511,0.5591306686401367
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,511,0.4837813377380371
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,1023,1.1178666750590007
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,float16,1,0.08444799979527791
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,1023,0.9166506926218668
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,fp8,1,0.27511467536290485
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,float16,3,0.08463999629020691
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,float16,7,0.08738133311271667
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,fp8,3,0.27477333943049115
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,fp8,7,0.290501336256663
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,float16,31,0.11505066355069478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,float16,15,0.09113599856694539
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,fp8,15,0.30481066306432086
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,fp8,31,0.39150933424631756
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,float16,63,0.14728533228238425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,fp8,63,0.39150933424631756
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,fp8,127,0.40375999609629315
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,float16,127,0.2880693276723226
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,float16,255,0.5620053211847941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,fp8,255,0.5155839920043945
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,float16,1,0.16486400365829468
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,fp8,1,0.5461333195368449
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,fp8,3,0.5454506476720175
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,float16,3,0.16503467162450156
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,float16,7,0.17117865880330405
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,fp8,7,0.5792320171991984
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,float16,15,0.17937066157658896
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,fp8,15,0.6038186550140381
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,float16,31,0.237226665019989
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,fp8,31,0.7760213216145834
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,float16,63,0.30133867263793945
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,fp8,63,0.7934292952219645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,float16,127,0.5686613321304321
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,fp8,127,0.7980213165283203
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,1,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,3,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,1,0.02935466667016347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,3,0.02935466667016347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,7,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,7,0.02918400118748347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,15,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,15,0.029365333418051403
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,31,0.009888000165422758
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,63,0.03703466554482778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,31,0.031744000812371574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,63,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,127,0.011717333147923151
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,127,0.037045332292715706
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,255,0.012410666793584824
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,255,0.03743999948104223
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,511,0.015301333119471868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,511,0.045909335215886436
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,1023,0.027136000494162243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,1023,0.077824001510938
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,2047,0.05085866649945577
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,2047,0.14199466506640115
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,4095,0.4899839957555135
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,4095,0.27613866329193115
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,8191,1.0130773385365803
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,8191,0.5620053211847941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,16383,1.1818666458129883
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,16383,2.1519360542297363
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,float16,1,0.32476266225179035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,float16,3,0.3252906600634257
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,fp8,1,1.0857813358306885
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,fp8,3,1.086074670155843
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,float16,7,0.34355199337005615
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,fp8,7,1.160703976949056
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,float16,15,0.38894931475321454
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,fp8,15,1.207103967666626
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,float16,31,0.47707200050354004
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,float16,63,0.5930666526158651
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,fp8,31,1.5684266090393066
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,fp8,63,1.5747413635253906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,float16,1,0.6536533435185751
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,fp8,1,2.166639963785807
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,float16,3,0.6867626508076986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,float16,7,0.7454346815745035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,fp8,3,2.1700266202290854
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,float16,15,0.783349355061849
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,fp8,7,2.371568044026693
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,1,0.009194666519761086
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,fp8,15,2.489855925242106
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,3,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,1,0.02070933332045873
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,float16,31,0.9494187037150065
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,3,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,7,0.009408000235756239
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,fp8,31,3.1320746739705405
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,7,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,15,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,31,0.010725333044926325
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,15,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,31,0.0315733328461647
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,63,0.010954666882753372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,63,0.0315733328461647
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,127,0.011930666863918304
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,127,0.031727999448776245
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,255,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,255,0.039408000806967415
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,511,0.025637333591779072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,511,0.07031466563542683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,1023,0.04619733492533366
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,1023,0.13209600249926248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,2047,0.4899839957555135
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,2047,0.25702399015426636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,4095,0.5509119828542074
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,4095,1.0861226717631023
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,1,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,8191,2.1466453870137534
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,8191,1.185477336247762
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,1,0.04215466479460398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,3,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,3,0.04351999859015147
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,7,0.014346666634082794
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,7,0.045567999283472695
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,15,0.017407999684413273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,15,0.05563733478387197
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,31,0.017621333400408428
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,31,0.05563733478387197
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,63,0.018266666680574417
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,63,0.05597866574923197
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,127,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,127,0.06980266670385997
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,255,0.04351999859015147
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,511,0.29320534070332843
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,255,0.12424533565839131
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,511,0.2367146611213684
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,1023,0.9808213710784912
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,1023,0.48657067616780597
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,2047,1.0064213275909424
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,2047,2.1473280588785806
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,1,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,1,0.007221333061655362
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,4095,4.338858604431152
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,3,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,4095,2.2087626457214355
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,3,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,7,0.013194666554530462
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,15,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,31,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,31,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,127,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,255,0.008687999720374743
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,255,0.019626667102177937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,511,0.012154666086037954
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,511,0.034645333886146545
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,1023,0.02128000060717265
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,1023,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,2047,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,2047,0.07816533247629802
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,4095,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,4095,0.14011733730634054
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,8191,0.07321600119272868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,8191,0.2563413381576538
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,16383,0.5753173430760702
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,1,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,16383,0.5022720098495483
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,3,0.007482666522264481
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,3,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,7,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,15,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,15,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,31,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,31,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,63,0.008687999720374743
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,63,0.014736000448465347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,127,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,127,0.008650666723648706
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,255,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,255,0.03310933212439219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,511,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,511,0.045567999283472695
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,1023,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,1023,0.06741333504517873
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,2047,0.03874133278926214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,2047,0.11212799946467082
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,4095,0.06178133189678192
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,8191,0.5015893379847208
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,8191,0.38365864753723145
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,4095,0.20121600230534872
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,1,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,16383,1.043455998102824
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,1,0.07970133423805237
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,16383,0.8084479967753092
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,3,0.025759999950726826
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,3,0.0828959991534551
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,7,0.02628266563018163
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,7,0.08584533135096233
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,15,0.032085334261258446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,15,0.10615467031796773
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,63,0.03328000009059906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,31,0.032255999743938446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,31,0.10615467031796773
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,63,0.10615467031796773
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,127,0.05495466788609823
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,127,0.13346133629480997
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,255,0.28757333755493164
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,255,0.24473599592844644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,511,0.567413330078125
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,511,0.4763306776682536
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,1023,2.0833279291788735
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,1023,0.9255253473917643
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,2047,1.9290453592936199
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,2047,4.225194613138835
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,1,0.0225600004196167
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,3,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,7,0.009930666536092758
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,7,0.02257599929968516
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,15,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,15,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,31,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,63,0.00847999999920527
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,31,0.0266239990790685
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,63,0.02679466704527537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,127,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,127,0.02679466704527537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,255,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,255,0.03276800115903219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,511,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,511,0.05394133428732554
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,1023,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,1023,0.09710933764775594
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,2047,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,2047,0.18295466899871826
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,4095,0.5410133202870687
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,4095,0.36130134264628094
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,8191,1.0840853055318196
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,8191,0.7741493384043375
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,1,0.04524266719818115
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,16383,2.1526400248209634
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,16383,1.5214932759602864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,1,0.14711466431617737
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,3,0.04693333307902018
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,3,0.15377066532770792
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,7,0.048810665806134544
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,7,0.16010666886965433
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,15,0.06144000093142191
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,15,0.20548266172409058
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,31,0.2053119937578837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,31,0.06111466884613037
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,63,0.12578133742014566
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,63,0.20599466562271118
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,127,0.2913279930750529
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,127,0.26845866441726685
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,255,0.5637120008468628
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,255,0.4901546637217204
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,511,1.1255466938018799
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,511,0.9236480394999186
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,1,0.0865280032157898
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,1023,4.18781852722168
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,1023,1.8027520179748535
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,fp8,1,0.2851840058962504
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,3,0.0897706647713979
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,fp8,3,0.29815467198689777
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,7,0.0936959981918335
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,fp8,7,0.3135733405749003
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,15,0.1181013286113739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,31,0.14320533474286398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,fp8,15,0.40140799681345624
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,fp8,31,0.4015786647796631
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,63,0.29730133215586346
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,fp8,63,0.41627200444539386
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,127,0.57314666112264
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,fp8,127,0.5275306701660156
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,255,1.1158186594645183
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,fp8,255,0.9565707047780355
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,float16,1,0.16793600718180338
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,fp8,1,0.5638826688130697
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,float16,3,0.17578667402267456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,fp8,3,0.5928959846496582
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,float16,7,0.18568533658981323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,fp8,7,0.6210560003916422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,float16,15,0.24644267559051514
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,float16,31,0.3298986752827962
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,fp8,15,0.7968426545461019
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,fp8,31,0.8128853638966879
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,float16,63,0.589141329129537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,fp8,63,0.8197120030721029
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,float16,127,1.1344213485717773
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,1,0.010053333515922228
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,fp8,127,1.0400426387786865
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,1,0.03276800115903219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,3,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,3,0.03243733445803324
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,7,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,7,0.03276800115903219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,15,0.010186666622757912
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,15,0.034474665919939675
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,31,0.011744000017642975
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,31,0.038917332887649536
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,63,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,63,0.03925333420435587
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,127,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,127,0.039594667653242745
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,255,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,255,0.04795733094215393
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,511,0.025600001215934753
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,511,0.07987200220425923
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,1023,0.05222400029500326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,1023,0.14455466469128928
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,2047,0.49032533168792725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,2047,0.28142933050791424
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,4095,1.0084693431854248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,4095,0.5676426490147909
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,8191,1.1876693566640217
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,8191,2.1814613342285156
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,16383,4.28817081451416
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,16383,2.3362560272216797
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,float16,1,0.3333119948705037
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,fp8,1,1.1192426681518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,float16,3,0.36420265833536786
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,fp8,3,1.1868000030517578
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,float16,7,0.40618666013081867
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,fp8,7,1.2615679899851482
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,float16,15,0.49544533093770343
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,fp8,15,1.6151893933614094
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,float16,31,0.6507519880930582
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,fp8,31,1.6165547370910645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,float16,63,1.1728213628133137
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,fp8,63,1.626794656117757
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,float16,1,0.746837298075358
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,float16,3,0.7690239747365316
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,fp8,1,2.331989288330078
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,float16,7,0.8091306686401367
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,fp8,3,2.466474692026774
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,float16,15,0.9849173227945963
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,fp8,7,2.562901337941488
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,fp8,15,3.2206506729125977
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,1,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,1,0.023210667073726654
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,3,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,float16,31,1.2967413266499836
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,3,0.023578666150569916
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,7,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,fp8,31,3.2247467041015625
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,7,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,15,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,15,0.032255999743938446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,31,0.010965333630641302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,31,0.032255999743938446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,63,0.011946666985750198
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,63,0.032255999743938446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,127,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,127,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,255,0.025600001215934753
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,255,0.07099733253320058
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,511,0.049839998284975685
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,511,0.13329066832860312
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,1023,0.48571733633677167
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,1023,0.2606079975763957
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,2047,0.5526346762975057
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,2047,0.9983999729156494
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,4095,2.1734399795532227
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,4095,1.1699199676513672
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,1,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,8191,4.289504051208496
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,8191,2.3678293228149414
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,1,0.03788800040880839
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,3,0.012821332861979803
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,3,0.03788800040880839
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,7,0.03940266619126002
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,7,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,15,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,31,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,15,0.040618665516376495
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,31,0.049322664737701416
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,63,0.04934399823347727
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,63,0.01586666703224182
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,127,0.01653333380818367
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,127,0.04937066634496053
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,255,0.02102400114138921
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,255,0.062122667829195656
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,511,0.037658666570981346
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,511,0.11212799946467082
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,1023,0.2455893357594808
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,1023,0.2126506765683492
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,2047,0.859818696975708
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,2047,0.4410346746444702
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,1,0.0074560002734263735
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,4095,1.8213547070821126
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,4095,0.9146026770273844
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,3,0.00702400008837382
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,3,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,15,0.007216000308593114
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,15,0.011061333119869232
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,31,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,31,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,63,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,63,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,127,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,127,0.01379199946920077
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,255,0.009365333244204521
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,255,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,511,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,511,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,1023,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,1023,0.018944000204404194
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,2047,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,2047,0.04384533564249674
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,4095,0.024271999796231587
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,4095,0.06534400085608165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,8191,0.03756266583998998
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,8191,0.10854400197664897
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,16383,0.06142933170000712
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,16383,0.1960960030555725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,3,0.007466666400432587
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,3,0.011045332998037338
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,7,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,7,0.01129066695769628
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,15,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,31,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,31,0.011946666985750198
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,63,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,63,0.013546666751305262
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,127,0.013797332843144735
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,255,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,255,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,511,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,511,0.031221332649389904
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,1023,0.019797333826621372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,1023,0.04369066655635834
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,2047,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,2047,0.06502399841944377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,4095,0.037205333511034645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,4095,0.10905599594116211
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,8191,0.059562668204307556
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,8191,0.19592533508936563
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,16383,0.45926400025685626
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,16383,0.36983466148376465
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,1,0.020469332734743755
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,1,0.06570666531721751
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,3,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,3,0.06570666531721751
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,7,0.021333334346612293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,7,0.0682666649421056
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,15,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,15,0.07065600156784058
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,31,0.027306665976842243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,31,0.08925867080688477
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,63,0.027647999425729115
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,63,0.08925867080688477
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,127,0.028330666323502857
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,127,0.08942932883898418
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,255,0.0365226666132609
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,255,0.11315199732780457
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,511,0.23944532871246338
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,1023,0.4747946659723918
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,511,0.20616533358891806
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,1023,0.4089173475901286
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,2047,1.7167359987894695
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,1,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,2047,0.7966720263163248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,1,0.019626667102177937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,3,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,3,0.019797333826621372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,7,0.010768000036478043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,7,0.019626667102177937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,15,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,15,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,31,0.021333334346612293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,31,0.011253333340088526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,63,0.012304000556468964
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,63,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,127,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,127,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,255,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,255,0.025600001215934753
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,511,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,511,0.03089066594839096
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,1023,0.019968000551064808
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,1023,0.05253866811593374
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,2047,0.032085334261258446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,2047,0.09506133198738098
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,4095,0.054272000988324486
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,4095,0.18141865730285645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,8191,0.4495360056559245
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,8191,0.35225598017374676
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,16383,0.9050453503926595
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,16383,0.7635626792907715
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,1,0.03754666695992152
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,1,0.12100266416867574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,3,0.037717332442601524
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,3,0.12113599975903828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,7,0.03878933439652125
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,7,0.1269599994023641
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,15,0.13175466656684875
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,15,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,31,0.05050133168697357
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,31,0.16895999511082968
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,63,0.050901333491007485
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,63,0.16913066307703653
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,127,0.05222400029500326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,127,0.1693013310432434
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,255,0.23997867107391357
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,255,0.21572266022364298
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,511,0.4679679870605469
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,511,0.40807998180389404
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,1023,0.9431040287017822
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,1023,0.7738026777903239
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,float16,1,0.07082666456699371
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,fp8,1,0.23197867472966513
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,float16,3,0.07133866846561432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,fp8,3,0.23244800170262656
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,float16,7,0.07355733215808868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,fp8,7,0.2461013396581014
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,float16,15,0.0766293356815974
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,float16,31,0.09642666578292847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,fp8,15,0.25565866629282635
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,float16,63,0.09762133161226909
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,fp8,31,0.32819199562072754
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,fp8,63,0.32870399951934814
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,float16,127,0.2409813404083252
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,fp8,127,0.32958932717641193
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,float16,255,0.47018667062123615
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,fp8,255,0.4317440191904704
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,float16,1,0.13755733768145242
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,fp8,1,0.4567199945449829
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,float16,3,0.13784533739089966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,fp8,3,0.45687464872996014
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,float16,7,0.14267733693122864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,fp8,7,0.4838399887084961
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,float16,15,0.14967466394106546
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,float16,31,0.19404266277949014
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,fp8,15,0.5048266649246216
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,fp8,31,0.6478506724039713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,float16,63,0.2525866627693176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,fp8,63,0.6490240097045898
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,float16,127,0.4751360019048055
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,fp8,127,0.6686506271362305
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,1,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,1,0.029365333418051403
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,3,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,3,0.02884799987077713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,7,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,7,0.02935466667016347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,15,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,15,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,31,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,31,0.0314026673634847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,63,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,63,0.03669333209594091
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,127,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,127,0.037205333511034645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,255,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,255,0.03703466554482778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,511,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,511,0.045909335215886436
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,1023,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,1023,0.07762666543324788
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,2047,0.043007999658584595
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,2047,0.14230400323867798
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,4095,0.26999467611312866
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,4095,0.44629331429799396
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,8191,0.8934400081634521
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,8191,0.559445341428121
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,16383,1.7298773129781086
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,16383,1.167189359664917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,float16,1,0.27002666393915814
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,fp8,1,0.9062399864196777
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,float16,3,0.27084799607594806
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,fp8,3,0.9067520300547282
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,float16,7,0.2815999984741211
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,fp8,7,0.9678506851196289
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,float16,15,0.31470932563145954
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,fp8,15,1.0048853556315105
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,float16,31,0.4007253249486287
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,fp8,31,1.2997759977976482
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,float16,63,0.49544533093770343
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,fp8,63,1.3144746621449788
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,float16,1,0.5350613196690878
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,float16,3,0.5502666632334391
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,fp8,1,1.8080426851908367
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,fp8,3,1.8097492853800456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,float16,7,0.611845334370931
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,fp8,7,1.951573371887207
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,float16,15,0.6536533435185751
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,1,0.008986666798591614
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,fp8,15,2.0711894035339355
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,1,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,float16,31,0.7920640309651693
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,3,0.009056000038981438
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,fp8,31,2.6102293332417807
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,3,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,7,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,7,0.021333334346612293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,15,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,15,0.009189333145817121
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,31,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,31,0.026799999177455902
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,63,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,63,0.026975999275843304
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,127,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,127,0.027136000494162243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,255,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,255,0.03379199902216593
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,511,0.023210667073726654
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,511,0.05938666562239329
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,1023,0.043178667624791466
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,1023,0.1109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,2047,0.4456106821695964
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,2047,0.2146986722946167
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,4095,0.854528029759725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,4095,0.46250665187835693
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,8191,1.731071949005127
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,1,0.013114667187134424
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,8191,1.0719573497772217
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,1,0.040778666734695435
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,3,0.013637332866589228
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,3,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,7,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,7,0.043007999658584595
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,15,0.015882667154073715
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,15,0.05053866902987162
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,31,0.05085866649945577
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,31,0.016042667130629223
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,63,0.016704000532627106
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,63,0.05085866649945577
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,127,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,127,0.06348800162474315
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,255,0.037717332442601524
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,255,0.11315199732780457
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,511,0.2486613392829895
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,511,0.21452800432840982
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,1023,0.861525297164917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,1023,0.4453973372777303
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,2047,1.7259519894917805
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,2047,0.9173333644866943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,1,0.007029333462317784
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,1,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,4095,2.032128016153971
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,3,0.01310933381319046
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,4095,3.6379305521647134
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,7,0.00749333327015241
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,7,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,15,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,31,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,31,0.008170666793982187
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,127,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,127,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,255,0.019626667102177937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,255,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,511,0.011957333733638128
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,511,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,1023,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,1023,0.045194665590922035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,2047,0.024746666351954143
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,2047,0.06724266707897186
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,4095,0.03669333209594091
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,4095,0.11127466956774394
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,8191,0.058880001306533813
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,8191,0.20002132654190063
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,16383,0.4557173252105713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,16383,0.3774720033009847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,1,0.007520000139872233
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,1,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,3,0.012986666212479273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,7,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,15,0.007631999750932057
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,15,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,31,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,31,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,63,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,127,0.008559999987483025
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,127,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,255,0.008879999940594038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,511,0.019973333925008774
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,255,0.03276800115903219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,511,0.045237332582473755
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,1023,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,1023,0.06708799799283345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,2047,0.0365226666132609
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,2047,0.11156266927719116
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,4095,0.20121600230534872
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,4095,0.05858133236567179
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,8191,0.45533867677052814
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,8191,0.37802668412526447
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,1,0.06911999980608623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,1,0.020992000897725422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,16383,0.9018026987711588
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,16383,0.8063999811808268
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,3,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,3,0.07133866846561432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,7,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,7,0.07389866809050243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,15,0.027647999425729115
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,15,0.091648002465566
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,63,0.02867199977238973
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,63,0.091648002465566
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,31,0.09130666653315227
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,31,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,127,0.03703466554482778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,127,0.115365336338679
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,255,0.24149332443873087
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,255,0.2082293430964152
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,511,0.41386667887369794
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,511,0.4790613253911336
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,1023,0.8028159936269125
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,1023,1.7201493581136067
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,2047,3.575637181599935
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,2047,1.6680960655212402
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,1,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,1,0.02218666672706604
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,3,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,3,0.0223786657055219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,7,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,7,0.022357332209746044
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,15,0.023743999501069386
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,31,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,31,0.0264533335963885
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,63,0.02679466704527537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,127,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,127,0.02679466704527537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,255,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,255,0.03242666771014532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,511,0.0194560003777345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,511,0.054101333022117615
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,1023,0.03242666771014532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,1023,0.09709866841634114
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,2047,0.05614933371543884
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,2047,0.183296004931132
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,4095,0.44999468326568604
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,4095,0.3559946616490682
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,8191,0.8898560206095377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,8191,0.7717546621958414
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,1,0.03857066730658213
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,16383,1.5293439229329426
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,1,0.12612266341845194
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,16383,1.806165377298991
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,3,0.040063999593257904
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,3,0.13173333803812662
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,7,0.04136000076929728
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,7,0.1365333298842112
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,15,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,15,0.17339734236399332
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,31,0.051882664362589516
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,31,0.17339734236399332
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,63,0.05273599922657013
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,63,0.17341333627700806
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,127,0.24452267090479532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,127,0.2193066676457723
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,255,0.41369601090749103
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,255,0.47244266668955487
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,511,0.7813119888305664
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,511,0.9320106506347656
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,1023,3.5581067403157554
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,1023,1.5209813117980957
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,1,0.07236266632874806
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,fp8,1,0.24064000447591147
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,3,0.07577600081761678
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,fp8,3,0.2518506646156311
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,7,0.07897600034872691
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,fp8,7,0.26265599330266315
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,15,0.09897599617640178
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,31,0.10308266679445903
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,fp8,15,0.336575984954834
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,fp8,31,0.33672531445821124
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,63,0.26129066944122314
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,fp8,63,0.3370399872461955
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,127,0.47832000255584717
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,fp8,127,0.44202665487925213
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,255,0.9326933224995931
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,fp8,255,0.8014240264892578
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,float16,1,0.14028799533843994
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,float16,3,0.14711466431617737
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,fp8,1,0.47121067841847736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,fp8,3,0.494762659072876
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,float16,7,0.1551359991232554
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,fp8,7,0.5184853474299113
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,float16,15,0.20155733823776245
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,fp8,15,0.6637226740519205
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,float16,31,0.274944007396698
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,fp8,31,0.6738080183664957
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,float16,63,0.5208746592203776
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,fp8,63,0.6857386430104574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,float16,127,0.9475413163503011
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,fp8,127,0.8704000314076742
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,1,0.009568000212311745
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,1,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,3,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,3,0.032085334261258446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,7,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,7,0.03187733391920725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,15,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,15,0.03379199902216593
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,31,0.01156266654531161
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,31,0.03873066604137421
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,63,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,63,0.038912000755469
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,127,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,127,0.03925333420435587
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,255,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,255,0.04778666794300079
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,511,0.024405332903067272
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,511,0.07937600215276082
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,1023,0.04334933559099833
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,1023,0.14405866463979086
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,2047,0.446122686068217
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,2047,0.2730666597684224
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,4095,0.8847359816233317
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,4095,0.5628586610158285
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,8191,1.7969600359598796
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,8191,1.1748692989349365
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,16383,3.582122802734375
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,16383,2.3176533381144204
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,float16,1,0.27562665939331055
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,float16,3,0.29474133253097534
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,fp8,1,0.9333759943644205
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,fp8,3,0.9878186384836832
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,float16,7,0.3327733278274536
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,fp8,7,1.037823994954427
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,float16,15,0.41420801480611164
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,fp8,15,1.3443412780761719
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,float16,31,0.5435413519541422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,fp8,31,1.349631945292155
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,float16,63,0.9779199759165446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,fp8,63,1.3568000793457031
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,float16,1,0.612010677655538
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,fp8,1,1.9188052813212078
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,float16,3,0.6382933457692465
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,fp8,3,2.052778720855713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,float16,7,0.6748159726460775
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,float16,15,0.8222933610280355
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,fp8,7,2.1382452646891275
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,fp8,15,2.685098648071289
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,1,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,1,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,float16,31,1.0777599811553955
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,3,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,3,0.023402666052182514
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,fp8,31,2.700079917907715
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,7,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,7,0.02441066751877467
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,15,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,15,0.028149334092934925
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,31,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,31,0.010458666831254959
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,63,0.01090666651725769
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,63,0.02802666773398717
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,127,0.013855999956528345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,127,0.034645333886146545
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,255,0.06058666606744131
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,255,0.023039999107519787
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,511,0.04369066655635834
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,511,0.11212799946467082
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,1023,0.21635200579961142
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,1023,0.44580264886220294
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,2047,0.8854506810506185
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,2047,0.46455466747283936
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,4095,1.8106026649475098
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,4095,0.9779199759165446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,1,0.009770666559537252
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,1,0.029696000119050343
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,8191,2.126789410909017
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,3,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,8191,3.583658536275228
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,3,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,7,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,7,0.0312266672650973
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,15,0.010437333335479101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,15,0.031744000812371574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,31,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,31,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,63,0.013343999783198038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,63,0.040448000033696495
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,127,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,127,0.040618665516376495
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,255,0.0170666662355264
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,255,0.05100266635417938
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,511,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,511,0.09066133697827657
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,1023,0.20036266247431436
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,1023,0.17220266660054526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,2047,0.7133866945902506
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,2047,0.35788798332214355
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,4095,1.354581356048584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,4095,0.7256746292114258
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,8191,1.577130635579427
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,8191,2.8637866973876953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,1,0.011440000186363855
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,3,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,3,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,7,0.011071999867757162
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,15,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,15,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,31,0.007354666789372762
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,31,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,63,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,63,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,127,0.008400000010927519
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,127,0.013631999492645264
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,255,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,255,0.013679999858140945
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,511,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,511,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,1023,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,1023,0.01826133330663045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,2047,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,2047,0.03256533294916153
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,4095,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,8191,0.03498666733503342
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,4095,0.053930665055910744
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,8191,0.09215999643007915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,16383,0.05681600173314413
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,16383,0.15769599874814352
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,32767,0.35020800431569415
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,32767,0.2940586606661479
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,1,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,3,0.0075626665105422335
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,3,0.011077333241701126
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,7,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,15,0.012778667112191519
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,63,0.008165333420038223
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,31,0.007536000261704127
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,31,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,63,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,127,0.013658666362365087
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,255,0.008346666892369589
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,255,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,511,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,511,0.018122666825850803
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,1023,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,1023,0.03265066693226496
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,2047,0.02218666672706604
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,2047,0.04849066833655039
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,4095,0.03292266776164373
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,4095,0.07645866771539052
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,8191,0.04778666794300079
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,8191,0.13597866892814636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,16383,0.21640533208847046
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,16383,0.25031999746958417
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,32767,0.6712319850921631
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,1,0.03362133353948593
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,32767,0.511674682299296
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,1,0.04896533489227295
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,3,0.0170666662355264
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,3,0.04915200173854828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,7,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,7,0.051029334465662636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,15,0.018272000054518383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,15,0.05444266895453135
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,31,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,31,0.07167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,63,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,63,0.0718506673971812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,127,0.023381332556406658
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,127,0.07202133536338806
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,255,0.030394665896892548
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,255,0.09113066395123799
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,511,0.19526932636896768
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,511,0.16450132926305136
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,1023,0.38600532213846844
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,1023,0.328874667485555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,2047,0.6410186688105265
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,2047,1.3436586062113445
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,4095,2.8842665354410806
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,4095,1.3380266825358074
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,1,0.00730666642387708
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,7,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,31,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,15,0.01108266661564509
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,31,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,63,0.008330666770537695
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,127,0.008522666369875273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,63,0.01360000049074491
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,255,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,511,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,255,0.01431999976436297
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,511,0.0315733328461647
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,1023,0.02144533395767212
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,1023,0.04399466514587402
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,2047,0.027285332481066387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,2047,0.06553600231806438
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,4095,0.043178667624791466
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,4095,0.10939733187357585
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,8191,0.2177706758181254
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,8191,0.1962666710217794
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,16383,0.7077546914418539
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,16383,0.3935199975967407
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,32767,1.4738772710164387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,32767,0.8094720045725504
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,1,0.031061333914597828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,1,0.09574400385220845
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,3,0.031061333914597828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,3,0.09574400385220845
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,7,0.0321066677570343
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,7,0.10001066327095032
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,15,0.03310933212439219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,31,0.040618665516376495
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,15,0.10393599669138591
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,63,0.04113066693147024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,31,0.13448533415794373
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,63,0.1346560021241506
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,127,0.04249600072701772
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,127,0.13482667009035745
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,255,0.19387733936309814
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,255,0.17151999473571777
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,511,0.37768534819285077
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,511,0.3264853358268738
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,1023,0.7475199699401855
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,1023,0.6138879855473837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,2047,2.852522532145182
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,2047,1.1973973115285237
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,1,0.0576853354771932
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,1,0.18705066045125326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,3,0.057855998476346336
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,3,0.18722132841746011
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,7,0.059903999169667564
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,7,0.19729065895080566
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,15,0.06200533111890157
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,15,0.20548266172409058
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,31,0.077824001510938
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,31,0.26316799720128375
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,63,0.07884799937407176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,63,0.26368000109990436
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,127,0.19541333119074503
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,127,0.2640213370323181
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,255,0.37802668412526447
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,255,0.3474773168563843
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,511,0.6364159981409708
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,511,0.7427519957224528
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,float16,1,0.11158399780591328
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,fp8,1,0.3662506739298503
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,float16,3,0.11161599556605022
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,fp8,3,0.3659093379974365
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,float16,7,0.11502933502197266
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,fp8,7,0.38690133889516193
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,float16,15,0.12066133817036946
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,float16,31,0.1525866687297821
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,fp8,15,0.40430935223897296
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,fp8,31,0.5198453267415365
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,float16,63,0.22289599974950156
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,fp8,63,0.520362655321757
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,fp8,127,0.5358933210372925
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,float16,127,0.3821226755777995
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,1,0.006826666494210561
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,float16,255,0.748032013575236
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,fp8,255,0.6843732992808024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,1,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,3,0.006826666494210561
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,3,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,7,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,7,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,15,0.007344000041484833
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,15,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,31,0.007840000092983246
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,31,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,63,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,63,0.025258667767047882
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,127,0.008341333518425623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,127,0.025418666501839954
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,255,0.025770666698614757
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,255,0.009093333035707474
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,511,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,511,0.031685332457224526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,1023,0.017237332959969837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,1023,0.05273599922657013
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,2047,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,2047,0.09540266791979472
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,4095,0.3614559968312581
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,4095,0.18107734123865762
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,8191,0.7290879885355631
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,8191,0.3752959966659546
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,16383,1.4340906143188477
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,16383,0.779263973236084
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,float16,1,0.2491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,32767,1.6399359703063965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,32767,2.9265918731689453
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,fp8,1,0.7261280218760172
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,float16,3,0.21913599967956543
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,fp8,3,0.7270399729410807
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,float16,7,0.22562134265899658
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,fp8,7,0.7748266855875651
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,float16,15,0.2450773318608602
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,fp8,15,0.8046933015187582
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,float16,31,0.32051199674606323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,fp8,31,1.0321919918060303
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,float16,63,0.3973120053609212
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,fp8,63,1.0545492966969807
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,float16,127,0.7574186325073242
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,fp8,127,1.0600106716156006
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,float16,1,0.43246932824452716
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,float16,3,0.4346880118052165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,fp8,1,1.4470826784769695
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,fp8,3,1.4482773145039876
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,float16,7,0.47393067677815753
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,fp8,7,1.5523840586344402
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,float16,15,0.5247999827067057
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,fp8,15,1.633109410603841
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,1,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,float16,31,0.636245330174764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,1,0.0204373337328434
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,fp8,31,2.0899839401245117
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,float16,63,0.7869813442230225
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,3,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,fp8,63,2.09715207417806
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,3,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,7,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,7,0.021162666380405426
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,15,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,15,0.021333334346612293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,31,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,31,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,63,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,63,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,127,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,255,0.014688000082969666
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,255,0.02922133356332779
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,511,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,511,0.05101866523424784
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,1023,0.031744000812371574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,1023,0.09471999605496724
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,2047,0.2027519941329956
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,2047,0.18243199586868286
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,4095,0.6986880302429199
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,4095,0.38075733184814453
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,8191,1.4510080019632976
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,8191,0.7828479607899984
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,1,0.01009599988659223
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,16383,2.861743927001953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,1,0.032255999743938446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,16383,2.2729387283325195
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,3,0.010469333579142889
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,3,0.03328000009059906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,7,0.010757333288590113
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,7,0.03480000048875809
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,15,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,15,0.04164266586303711
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,31,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,63,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,31,0.04181333382924398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,63,0.04161600023508072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,127,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,127,0.05205333232879639
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,255,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,255,0.09198932846387227
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,511,0.20172800620396933
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,511,0.17442133029301962
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,1023,0.6956373055775961
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,1023,0.3597653309504191
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,2047,1.4363306363423665
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,2047,0.7294293244679769
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,1,0.007322666545708974
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,4095,1.5680853525797527
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,4095,2.8963839213053384
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,1,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,8191,3.1652212142944336
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,8191,5.7427628835042315
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,3,0.007301333049933116
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,3,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,7,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,15,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,31,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,31,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,63,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,255,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,127,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,255,0.019600000232458115
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,511,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,511,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,1023,0.012965332716703415
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,1023,0.03478399912516276
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,2047,0.022015998760859173
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,2047,0.05649066468079885
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,4095,0.0943946639696757
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,4095,0.0339626669883728
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,8191,0.05407999952634176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,8191,0.1621333360671997
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,16383,0.3556693394978841
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,16383,0.3006933331489563
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,1,0.007280000175038974
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,1,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,32767,0.7190186977386475
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,32767,0.6439253489176432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,3,0.007301333049933116
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,3,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,7,0.007381333038210869
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,7,0.013471999516089758
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,15,0.007631999750932057
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,15,0.013487999637921652
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,31,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,31,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,63,0.008522666369875273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,63,0.01481066644191742
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,127,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,255,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,255,0.019797333826621372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,511,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,511,0.034815999368826546
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,1023,0.022357332209746044
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,1023,0.05120000243186951
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,2047,0.031744000812371574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,2047,0.07850666840871175
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,4095,0.04693333307902018
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,4095,0.14040533701578775
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,8191,0.21504000822703043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,8191,0.2566986680030823
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,16383,0.6657280127207438
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,16383,0.5179733435312907
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,32767,1.4660266240437825
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,32767,1.0543786684672039
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,1,0.017423999806245167
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,1,0.05205333232879639
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,3,0.01791999985774358
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,3,0.053930665055910744
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,7,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,7,0.056661332647005715
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,15,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,31,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,15,0.07406933108965556
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,31,0.07401599983374278
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,63,0.07406933108965556
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,63,0.023552000522613525
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,127,0.03089066594839096
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,127,0.09284266829490662
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,255,0.24757866064707437
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,255,0.16622933745384216
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,511,0.38809601465861004
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,511,0.333679993947347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,1023,0.6765226523081461
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,1023,1.343488057454427
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,2047,1.3393920262654622
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,2047,2.8663466771443686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,1,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,1,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,4095,2.9803520838419595
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,3,0.013290667285521826
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,4095,5.784575780232747
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,7,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,7,0.013365333278973898
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,15,0.007653333246707916
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,15,0.01369599997997284
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,31,0.014655999839305878
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,31,0.008330666770537695
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,63,0.008858666444818178
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,127,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,127,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,255,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,255,0.033589333295822144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,511,0.04607999821503957
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,511,0.021503999829292297
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,1023,0.027136000494162243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,1023,0.0682666649421056
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,2047,0.04113066693147024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,2047,0.11332266529401143
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,4095,0.20206934213638306
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,4095,0.21640533208847046
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,8191,0.7094613711039225
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,8191,0.3991893529891968
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,16383,0.8158079783121744
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,16383,1.4539093971252441
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,32767,1.5880533854166667
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,32767,2.9197654724121094
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,1,0.031925333042939506
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,1,0.10018133123715718
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,3,0.03313066562016805
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,3,0.10410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,7,0.0341333324710528
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,7,0.10820266604423523
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,15,0.04153066625197729
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,15,0.13823999961217245
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,31,0.04181333382924398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,31,0.13823999961217245
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,63,0.042837331692377724
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,63,0.13871999581654867
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,127,0.1962666710217794
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,127,0.17425066232681274
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,255,0.38155198097229004
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,255,0.33023999134699505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,511,0.7533226807912191
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,511,0.6205173333485922
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,1023,1.2057600021362305
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,1023,2.857301394144694
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,1,0.05905066430568695
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,2047,2.5048747062683105
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,1,0.194432000319163
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,2047,5.7039947509765625
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,3,0.06144000093142191
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,3,0.20360533396402994
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,7,0.06384533147017162
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,7,0.21130667130152384
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,15,0.07987200220425923
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,31,0.08055466910203297
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,15,0.2720426718393962
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,31,0.26999467611312866
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,63,0.20104533433914185
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,63,0.27050666014353436
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,127,0.3838293155034383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,127,0.35549867153167725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,255,0.7502133051554362
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,255,0.6430186827977499
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,511,1.4824105898539226
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,511,1.2115626335144043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,1,0.11332266529401143
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,fp8,1,0.3773546616236369
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,3,0.1186133325099945
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,fp8,3,0.3973120053609212
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,7,0.12458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,fp8,7,0.41523198286692303
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,15,0.15889066457748413
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,fp8,15,0.5329920053482056
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,31,0.22033067544301352
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,fp8,31,0.5328213373819987
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,63,0.39458131790161133
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,fp8,63,0.5505493481953939
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,127,0.7589546839396158
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,fp8,127,0.6982133388519287
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,fp8,255,1.2683946291605632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,255,1.4871892929077148
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,1,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,1,0.02334933231274287
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,3,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,3,0.02319466571013133
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,7,0.023210667073726654
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,15,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,15,0.02405333270629247
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,31,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,31,0.027136000494162243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,63,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,63,0.026965332527955372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,127,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,127,0.027306665976842243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,255,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,255,0.03276800115903219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,511,0.0170666662355264
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,511,0.05461333195368449
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,1023,0.02935466667016347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,1023,0.09745066364606221
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,2047,0.36232535044352215
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,2047,0.1838080088297526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,4095,0.7191893259684244
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,4095,0.3797333240509033
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,8191,1.4551307360331218
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,8191,0.78438401222229
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,16383,1.536512056986491
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,16383,2.86737060546875
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,32767,5.795498530069987
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,float16,1,0.22203733523686728
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,32767,3.222698529561361
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,fp8,1,0.7492000261942545
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,float16,3,0.233130673567454
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,fp8,3,0.7912106513977051
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,float16,7,0.2578986684481303
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,fp8,7,0.8268799781799316
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,float16,15,0.3324586749076843
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,fp8,15,1.0697387059529622
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,float16,31,0.4352000157038371
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,fp8,31,1.0811200141906738
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,float16,63,0.7823359966278076
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,fp8,63,1.0885012944539387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,float16,127,1.5092213948567708
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,fp8,127,1.3847893079121907
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,float16,1,0.4695039987564087
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,float16,3,0.5099519888559977
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,fp8,1,1.4981013933817546
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,float16,7,0.5416959921518961
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,fp8,3,1.6184320449829102
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,fp8,7,1.708016077677409
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,float16,15,0.659231980641683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,fp8,15,2.1509119669596353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,1,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,1,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,float16,31,0.8634026845296224
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,float16,63,1.5575040181477864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,3,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,fp8,31,2.1519625981648765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,3,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,fp8,63,2.1638827323913574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,7,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,7,0.023381332556406658
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,15,0.02492800106604894
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,31,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,31,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,63,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,63,0.025258667767047882
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,127,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,127,0.030720000465710957
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,255,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,255,0.05239466826121012
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,511,0.032085334261258446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,511,0.09540266791979472
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,1023,0.2027519941329956
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,1023,0.18346667289733887
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,2047,0.7098026275634766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,2047,0.3829760154088338
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,4095,0.919381300608317
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,4095,1.4359893798828125
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,8191,1.5655253728230794
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,8191,2.907477378845215
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,1,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,16383,4.49843184153239
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,16383,5.726890563964844
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,1,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,3,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,3,0.02094399929046631
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,7,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,7,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,15,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,31,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,15,0.023039999107519787
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,31,0.0314026673634847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,63,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,63,0.0315733328461647
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,127,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,127,0.031744000812371574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,255,0.015013333410024643
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,255,0.03973866750796636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,511,0.025941332181294758
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,511,0.06997333467006683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,1023,0.04693333307902018
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,1023,0.13221333424250284
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,2047,0.2568800052007039
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,2047,0.48760000864664715
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,4095,1.084928035736084
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,4095,0.5507413148880005
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,1,0.011600000162919363
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,8191,2.148010730743408
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,8191,1.1839146614074707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,7,0.007135999699433644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,15,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,15,0.011653333902359009
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,31,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,31,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,63,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,63,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,255,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,255,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,511,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,511,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,1023,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,1023,0.01826133330663045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,2047,0.012495999534924826
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,2047,0.018944000204404194
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,4095,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,4095,0.048469334840774536
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,8191,0.030207999050617218
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,8191,0.07509333391984303
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,16383,0.04459733267625173
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,16383,0.13581867019335428
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,32767,0.07406933108965556
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,32767,0.2500266631444295
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,1,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,3,0.007146666447321574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,7,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,15,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,31,0.00766933336853981
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,31,0.012063999970753988
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,15,0.011600000162919363
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,63,0.008010666817426682
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,63,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,127,0.013834666460752487
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,255,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,255,0.008517333616813024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,511,0.008863999818762144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,511,0.01791999985774358
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,1023,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,1023,0.032085334261258446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,2047,0.020992000897725422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,2047,0.048298666874567665
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,4095,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,4095,0.07543466488520305
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,8191,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,8191,0.1360213359196981
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,16383,0.0769706666469574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,16383,0.24934399127960205
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,32767,0.5857280095418295
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,32767,0.4870826800664266
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,1,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,1,0.03942399968703588
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,3,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,3,0.03925333420435587
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,7,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,7,0.04095999896526337
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,15,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,15,0.04215466479460398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,31,0.017237332959969837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,63,0.034304000437259674
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,31,0.05415999889373779
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,63,0.05423999826113383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,127,0.018101333330074947
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,127,0.05461333195368449
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,255,0.023552000522613525
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,255,0.06843733290831248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,511,0.04454400142033895
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,511,0.1230506698290507
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,1023,0.2916693290074666
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,1023,0.233130673567454
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,2047,0.9804800351460775
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,2047,0.48315731684366864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,1,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,4095,2.15773868560791
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,4095,0.9927519957224528
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,1,0.011306667079528173
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,3,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,15,0.01108266661564509
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,31,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,31,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,63,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,63,0.0081386665503184
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,127,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,255,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,255,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,511,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,511,0.031231999397277832
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,1023,0.02014933278163274
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,1023,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,2047,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,2047,0.06502399841944377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,4095,0.03874133278926214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,4095,0.10871466994285583
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,8191,0.06519466638565063
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,8191,0.19659733772277832
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,16383,0.5073920090993246
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,16383,0.37646400928497314
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,32767,1.053866704305013
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,1,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,32767,0.794111967086792
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,1,0.07526400188604991
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,3,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,3,0.07543466488520305
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,7,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,7,0.07918933530648549
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,15,0.025600001215934753
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,15,0.08245866497357686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,31,0.0315733328461647
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,31,0.10274133086204529
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,63,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,63,0.1032480001449585
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,127,0.03293866664171219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,127,0.1032533347606659
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,255,0.045567999283472695
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,255,0.13124266266822815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,511,0.2397866646448771
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,511,0.2850133379300435
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,1023,0.5640533367792765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,1023,0.47121067841847736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,2047,0.9185280005137125
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,2047,2.049525260925293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,1,0.04437333345413208
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,1,0.1413279970486959
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,3,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,3,0.14148267110188803
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,7,0.04596266647179922
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,7,0.14865066607793173
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,15,0.04760533571243286
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,15,0.1551359991232554
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,31,0.05973333120346069
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,31,0.20036266247431436
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,63,0.060122668743133545
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,127,0.0766293356815974
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,63,0.20053333044052124
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,127,0.20116267601648966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,255,0.2860373258590698
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,255,0.27374933163324994
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,511,0.4838399887084961
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,511,0.5589333375295004
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,float16,1,0.0846506655216217
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,fp8,1,0.27477333943049115
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,float16,3,0.08499200145403545
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,fp8,3,0.27543999751408893
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,float16,7,0.08738133311271667
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,fp8,7,0.29097600777943927
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,float16,15,0.09148800373077393
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,float16,31,0.1151626706123352
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,fp8,15,0.3051519989967346
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,fp8,31,0.39104000727335614
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,float16,63,0.14966400464375815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,fp8,63,0.39190399646759033
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,fp8,127,0.4031146764755249
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,float16,127,0.288266658782959
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,1,0.006826666494210561
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,float16,255,0.5618346532185873
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,1,0.019797333826621372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,3,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,fp8,255,0.5498666763305664
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,3,0.006655999769767125
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,7,0.006826666494210561
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,7,0.019797333826621372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,15,0.006805333619316419
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,15,0.019968000551064808
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,31,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,31,0.007520000139872233
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,63,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,63,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,127,0.008010666817426682
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,127,0.025258667767047882
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,255,0.025429333249727886
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,511,0.010048000141978264
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,511,0.031061333914597828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,1023,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,1023,0.05239466826121012
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,2047,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,2047,0.09523199995358785
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,4095,0.1812480092048645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,4095,0.04795733094215393
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,8191,0.5442399978637695
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,8191,0.35310932000478107
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,16383,0.9970346291859945
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,16383,0.7632213433583578
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,32767,2.2036479314168296
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,32767,1.2260693709055583
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,float16,1,0.16605866948763529
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,fp8,1,0.5459626515706381
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,float16,3,0.16605866948763529
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,fp8,3,0.5461333195368449
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,float16,7,0.17032533884048462
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,fp8,7,0.5811200141906738
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,float16,15,0.17971199750900269
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,fp8,15,0.6048053503036499
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,float16,31,0.23859200874964395
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,fp8,31,0.7758506933848063
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,float16,63,0.30241066217422485
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,fp8,63,0.7929173310597738
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,float16,127,0.5683199961980184
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,fp8,127,0.8162986437479655
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,float16,1,0.32732266187667847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,float16,3,0.3280213276545207
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,fp8,1,1.0864266554514568
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,float16,7,0.3408213456471761
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,fp8,3,1.087829351425171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,float16,15,0.3899199962615967
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,fp8,7,1.1641173362731934
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,fp8,15,1.2062719662984211
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,1,0.009904000287254652
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,1,0.029525332152843475
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,3,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,float16,31,0.4792319933573405
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,fp8,31,1.5697919527689617
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,float16,63,0.5961386760075887
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,3,0.029525332152843475
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,fp8,63,1.5766186714172363
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,7,0.009925333162148794
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,7,0.02918400118748347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,15,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,15,0.02935466667016347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,31,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,31,0.031514666974544525
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,63,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,63,0.03737599899371465
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,127,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,255,0.01231466606259346
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,127,0.036864000062147774
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,255,0.03737599899371465
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,511,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,511,0.045738667249679565
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,1023,0.027306665976842243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,1023,0.07833600044250488
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,2047,0.051882664362589516
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,2047,0.14199466506640115
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,4095,0.4877653519312541
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,4095,0.27172799905141193
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,8191,1.012394666671753
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,8191,0.5626879930496216
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,16383,2.152613321940104
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,16383,1.1787947018941243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,1,0.009365333244204521
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,1,0.023045333723227184
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,3,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,3,0.023552000522613525
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,7,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,7,0.024746666351954143
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,15,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,15,0.03242666771014532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,31,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,31,0.032255999743938446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,63,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,63,0.03242666771014532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,127,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,127,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,255,0.07116800049940745
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,255,0.025770666698614757
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,511,0.04827733337879181
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,511,0.13329066832860312
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,1023,0.4843519926071167
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,1023,0.2595840096473694
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,2047,0.9994239807128906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,2047,0.552618662516276
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,4095,1.1574613253275554
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,4095,2.172074635823568
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,1,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,8191,4.29038937886556
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,8191,2.371936003367106
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,3,0.013104000439246496
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,7,0.013295999417702356
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,7,0.007157333195209503
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,15,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,31,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,31,0.014869333555301031
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,63,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,63,0.014688000082969666
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,127,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,127,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,255,0.008378666515151659
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,255,0.019306667149066925
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,511,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,511,0.020341333001852036
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,1023,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,1023,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,2047,0.01313599944114685
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,2047,0.05085866649945577
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,4095,0.030207999050617218
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,4095,0.07799466451009114
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,8191,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,8191,0.14011733730634054
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,16383,0.07526400188604991
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,16383,0.25596799453099567
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,32767,0.5753173430760702
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,1,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,32767,0.502784013748169
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,3,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,7,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,7,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,15,0.0074560002734263735
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,15,0.013669333110253016
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,31,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,31,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,127,0.00884799969693025
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,255,0.019626667102177937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,511,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,511,0.034304000437259674
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,1023,0.021312000850836437
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,1023,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,2047,0.030207999050617218
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,2047,0.07799466451009114
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,4095,0.07590400179227193
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,4095,0.14045866330464682
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,8191,0.07253333429495494
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,8191,0.2561440070470174
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,16383,0.5758293469746908
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,16383,0.5019306739171346
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,32767,1.151146650314331
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,32767,1.050112009048462
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,1,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,1,0.042037333051363625
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,3,0.014314666390419006
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,3,0.04351999859015147
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,7,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,7,0.045567999283472695
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,15,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,15,0.05563733478387197
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,31,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,31,0.05584000051021576
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,63,0.0183999997874101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,63,0.05597866574923197
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,127,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,127,0.06963199873765309
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,255,0.04369066655635834
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,255,0.12424533565839131
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,511,0.29263466596603394
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,511,0.23654399315516153
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,1023,0.9743359883626302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,1023,0.4864000082015991
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,2047,0.9972053368886312
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,2047,2.148010730743408
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,4095,4.338176091512044
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,1,0.0074560002734263735
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,1,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,4095,2.210474650065104
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,3,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,7,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,7,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,15,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,15,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,63,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,31,0.014645333091417948
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,127,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,127,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,255,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,255,0.03310933212439219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,511,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,511,0.045567999283472695
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,1023,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,1023,0.06758399804433186
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,2047,0.03760000069936117
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,2047,0.11181333661079407
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,4095,0.06173333525657654
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,4095,0.20172800620396933
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,8191,0.5050026575724283
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,8191,0.3839999834696452
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,16383,1.0356053511301677
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,16383,0.8055466810862223
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,32767,1.5680853525797527
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,32767,2.200064023335775
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,1,0.024746666351954143
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,1,0.0795306662718455
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,3,0.025941332181294758
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,3,0.08294400076071422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,7,0.0264533335963885
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,7,0.08584533135096233
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,15,0.032255999743938446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,15,0.10601066549619038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,31,0.032272001107533775
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,31,0.10615467031796773
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,63,0.03345066557327906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,63,0.1063253382841746
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,127,0.06365866462389629
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,127,0.13329066832860312
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,255,0.24490666389465332
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,255,0.2868640025456746
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,511,0.4761600097020467
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,511,0.5677706797917684
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,1023,2.052778720855713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,1023,0.9260373115539551
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,2047,4.243797302246094
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,2047,1.9253172874450684
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,1,0.04535999894142151
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,3,0.04693333307902018
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,1,0.14762666821479797
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,3,0.15377066532770792
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,7,0.04898133377234141
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,7,0.16044800480206808
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,15,0.06109866499900818
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,15,0.20565332969029745
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,31,0.06132266422112783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,31,0.20565332969029745
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,63,0.12731732924779257
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,63,0.20565332969029745
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,127,0.2918399969736735
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,127,0.2710026701291402
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,255,0.563749353090922
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,255,0.5072213411331177
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,511,0.9245013395945231
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,1,0.08669867118199666
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,511,1.1220160325368245
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,fp8,1,0.28484266996383667
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,3,0.09011200070381165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,fp8,3,0.29815467198689777
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,7,0.0942080020904541
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,fp8,7,0.3135146697362264
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,15,0.11878400047620137
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,fp8,31,0.40123732884724933
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,31,0.1437013347943624
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,fp8,15,0.40140799681345624
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,63,0.2974826693534851
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,fp8,63,0.4169386625289917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,127,0.5722453196843466
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,1,0.007040000210205714
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,fp8,127,0.5271893342336019
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,1,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,255,1.1168533166249592
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,fp8,255,0.9548799991607666
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,3,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,7,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,7,0.022757334013779957
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,15,0.02370133250951767
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,31,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,31,0.0266239990790685
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,63,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,63,0.02679466704527537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,127,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,127,0.026965332527955372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,255,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,255,0.03262399882078171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,511,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,511,0.054101333022117615
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,1023,0.024405332903067272
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,1023,0.09710933764775594
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,2047,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,2047,0.183296004931132
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,4095,0.36215468247731525
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,4095,0.5410133202870687
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,8191,0.7734613418579102
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,8191,1.0847573280334473
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,16383,2.1580586433410645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,16383,1.5232000350952148
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,float16,1,0.1687893271446228
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,32767,4.348245302836101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,fp8,1,0.5637120008468628
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,32767,2.4128853480021157
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,float16,3,0.1753013332684835
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,fp8,3,0.5934079885482788
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,float16,7,0.18756266434987387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,fp8,7,0.6212053298950195
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,float16,15,0.24727465709050497
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,float16,31,0.3322880069414775
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,fp8,15,0.7965013186136881
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,fp8,31,0.8108426729838053
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,float16,63,0.5879466533660889
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,fp8,63,0.8195412953694662
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,float16,127,1.1357706387837727
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,fp8,127,1.0419200261433919
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,float16,1,0.33604268232981366
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,float16,3,0.3657386700312297
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,fp8,1,1.1207679907480876
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,float16,7,0.408463994661967
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,fp8,3,1.18887464205424
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,fp8,7,1.262453317642212
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,float16,15,0.49715201059977215
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,fp8,15,1.6155306498209636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,1,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,1,0.03276800115903219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,float16,31,0.649727980295817
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,3,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,float16,63,1.170261303583781
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,3,0.03242666771014532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,fp8,31,1.6168959935506184
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,fp8,63,1.6274773279825847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,7,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,7,0.03259733319282532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,15,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,15,0.03428266694148382
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,31,0.039077334105968475
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,31,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,63,0.011834666132926941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,63,0.03908266623814901
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,127,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,127,0.039594667653242745
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,255,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,255,0.04795733094215393
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,511,0.028837333122889202
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,511,0.08004266520341237
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,1023,0.051029334465662636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,1023,0.14455466469128928
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,2047,0.48930132389068604
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,2047,0.27938133478164673
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,4095,0.5678079922993978
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,4095,1.008639971415202
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,8191,1.18067200978597
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,8191,2.181290626525879
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,1,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,16383,2.335573355356852
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,16383,4.287301381429036
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,1,0.020314666132132213
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,3,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,3,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,7,0.021333334346612293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,7,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,15,0.010405333091815313
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,15,0.021333334346612293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,31,0.011285333583752314
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,31,0.0339626669883728
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,63,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,63,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,127,0.014005333185195923
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,127,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,255,0.01479999969402949
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,255,0.02935466667016347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,511,0.01876266673207283
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,511,0.051029334465662636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,1023,0.031744000812371574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,1023,0.09471999605496724
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,2047,0.20309333006540933
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,2047,0.18225600322087607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,4095,0.6949546337127686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,4095,0.3814133405685425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,8191,1.4528853098551433
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,8191,0.7831892967224121
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,16383,2.2777172724405923
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,16383,2.861093203226725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,1,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,1,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,3,0.011642667154471079
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,3,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,7,0.007146666447321574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,15,0.011136000355084738
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,31,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,31,0.012282667060693106
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,63,0.008133333176374435
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,63,0.013829333086808523
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,127,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,255,0.009066666786869368
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,255,0.013776000589132309
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,511,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,511,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,1023,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,1023,0.01826133330663045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,2047,0.012400000045696894
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,2047,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,4095,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,4095,0.032474666833877563
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,8191,0.021162666380405426
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,8191,0.05395199855168661
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,16383,0.03310933212439219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,16383,0.09542399644851685
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,32767,0.053077335158983864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,32767,0.17851734161376953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,65535,0.3824640115102132
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,1,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,1,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,65535,0.33075199524561566
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,3,0.007333333293596904
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,7,0.007162666569153468
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,31,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,31,0.011749333391586939
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,63,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,63,0.0138026662170887
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,127,0.008341333518425623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,127,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,255,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,255,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,511,0.00873066671192646
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,511,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,1023,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,1023,0.01826133330663045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,2047,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,2047,0.03224000086386999
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,4095,0.021503999829292297
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,4095,0.054272000988324486
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,8191,0.0339626669883728
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,8191,0.09176533420880635
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,16383,0.05586666862169901
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,16383,0.157642662525177
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,32767,0.34628268082936603
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,32767,0.2935466567675273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,65535,0.7290879885355631
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,1,0.029525332152843475
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,1,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,3,0.009866666669646898
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,65535,0.6312959988911947
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,3,0.029738667110602062
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,7,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,7,0.03089066594839096
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,15,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,15,0.0315733328461647
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,31,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,31,0.04031466692686081
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,63,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,63,0.040618665516376495
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,127,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,127,0.040448000033696495
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,255,0.0170666662355264
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,255,0.0506879985332489
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,511,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,511,0.09079466263453166
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,1023,0.19780266284942627
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,1023,0.17220266660054526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,2047,0.7098293304443359
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,2047,0.35839998722076416
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,4095,1.3550933202107747
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,4095,0.7248000303904215
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,1,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,1,0.010970667004585266
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,8191,1.5766186714172363
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,8191,2.86737060546875
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,3,0.007280000175038974
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,3,0.011071999867757162
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,7,0.00731733317176501
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,15,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,31,0.0075573331365982694
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,31,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,63,0.00814933329820633
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,63,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,127,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,127,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,255,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,255,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,511,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,511,0.01791999985774358
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,1023,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,1023,0.032255999743938446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,2047,0.022015998760859173
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,2047,0.04881600042184194
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,4095,0.03293866664171219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,4095,0.07548800110816956
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,8191,0.048298666874567665
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,8191,0.1360213359196981
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,16383,0.22528000672658285
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,16383,0.2500266631444295
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,32767,0.6719146569569906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,32767,0.5116586685180664
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,1,0.016895999511082966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,65535,1.475935935974121
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,1,0.04898133377234141
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,65535,0.8118613560994467
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,3,0.016895999511082966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,3,0.04898133377234141
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,7,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,7,0.05120000243186951
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,15,0.0182239996890227
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,15,0.05461333195368449
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,31,0.022357332209746044
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,31,0.07167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,63,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,127,0.023381332556406658
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,127,0.07202133536338806
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,63,0.07202133536338806
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,255,0.030234667162100475
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,255,0.09096533060073853
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,511,0.1955839991569519
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,511,0.16435199975967407
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,1023,0.38519465923309326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,1023,0.3326293428738912
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,2047,1.3404266039530437
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,2047,0.6411946614583334
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,1,0.031061333914597828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,4095,2.8890721003214517
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,4095,1.3335893948872883
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,1,0.09572800000508626
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,3,0.031061333914597828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,3,0.09574400385220845
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,7,0.032085334261258446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,7,0.10018133123715718
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,15,0.03293866664171219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,15,0.10393599669138591
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,31,0.13431466619173685
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,31,0.04065066576004028
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,63,0.04113066693147024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,63,0.13482667009035745
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,127,0.04249600072701772
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,127,0.13516799608866373
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,255,0.19387733936309814
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,255,0.17151999473571777
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,511,0.37700267632802326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,511,0.3259733319282532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,1023,0.7470080057779948
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,1023,0.6145706574122111
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,1,0.057855998476346336
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,1,0.18705066045125326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,3,0.18722132841746011
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,3,0.05804799993832906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,7,0.05973866581916809
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,7,0.19746132691701254
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,15,0.06214933097362518
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,15,0.20614399512608847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,31,0.077824001510938
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,31,0.26316799720128375
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,63,0.07884799937407176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,63,0.2633440097173055
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,127,0.19503466288248697
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,127,0.26419200499852497
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,255,0.3784266710281372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,255,0.34815998872121173
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,511,0.7432373364766439
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,511,0.6364320119222006
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,1,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,1,0.0075519997626543045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,3,0.007477333148320516
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,3,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,7,0.007194666812817256
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,15,0.007471999774376552
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,31,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,31,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,15,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,63,0.008853333070874214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,63,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,127,0.008709333216150602
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,127,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,255,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,511,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,255,0.014186666657527288
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,511,0.031231999397277832
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,1023,0.021162666380405426
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,1023,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,2047,0.027141332626342773
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,2047,0.06553600231806438
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,4095,0.042725334564844765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,4095,0.10922666390736897
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,8191,0.21811199188232422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,8191,0.1962933341662089
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,16383,0.39321601390838623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,16383,0.6938933531443278
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,32767,1.4737067222595215
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,32767,0.8091306686401367
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,65535,2.950314521789551
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,65535,1.633962631225586
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,float16,1,0.11195733149846394
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,fp8,1,0.3658986488978068
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,float16,3,0.11229866743087769
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,fp8,3,0.3659093379974365
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,float16,7,0.11520000298817952
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,fp8,7,0.38741334279378253
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,float16,15,0.1204853355884552
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,fp8,15,0.40482131640116376
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,float16,31,0.15309866269429526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,float16,63,0.20258132616678873
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,fp8,31,0.5196799834569296
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,fp8,63,0.5198506514231364
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,float16,127,0.3824640115102132
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,fp8,127,0.5364053249359131
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,float16,255,0.7478613058725992
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,fp8,255,0.6840853691101074
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,float16,1,0.21964800357818604
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,fp8,1,0.726698637008667
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,float16,3,0.22016000747680664
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,fp8,3,0.7268693447113037
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,float16,7,0.22579199075698853
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,fp8,7,0.7762186527252197
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,fp8,15,0.8055466810862223
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,float16,15,0.24217599630355835
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,float16,31,0.32103999455769855
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,fp8,31,1.032538652420044
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,float16,63,0.3977973461151123
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,fp8,63,1.05348801612854
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,float16,127,0.7601493199666342
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,1,0.006826666494210561
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,1,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,fp8,127,1.0593279997507732
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,3,0.006826666494210561
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,3,0.02029866725206375
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,7,0.020138667275508244
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,15,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,31,0.021503999829292297
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,31,0.00772266648709774
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,63,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,63,0.025263999899228413
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,127,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,127,0.025301332275072735
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,255,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,255,0.025941332181294758
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,511,0.01089599976936976
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,1023,0.05256533126036326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,511,0.031231999397277832
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,2047,0.02918400118748347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,2047,0.09540266791979472
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,1023,0.017237332959969837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,4095,0.3838293155034383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,4095,0.18193066120147705
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,8191,0.7294293244679769
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,8191,0.37460800011952716
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,16383,0.7738026777903239
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,16383,1.434282620747884
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,1,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,32767,2.9332478841145835
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,1,0.022357332209746044
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,32767,1.6293546358744304
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,3,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,3,0.023039999107519787
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,7,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,7,0.023552000522613525
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,15,0.025066666305065155
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,31,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,63,0.013946666071812311
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,63,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,31,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,127,0.014709333578745524
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,127,0.03054933249950409
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,255,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,255,0.05205333232879639
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,511,0.03177600105603536
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,511,0.09540266791979472
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,1023,0.2032639980316162
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,1023,0.18363734086354574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,2047,0.3850239912668864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,2047,0.7091200351715088
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,4095,0.9345706303914388
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,4095,1.43394136428833
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,8191,2.907306671142578
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,8191,1.5689387321472168
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,1,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,16383,5.716480255126953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,3,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,7,0.007370666911204656
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,16383,4.506282806396484
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,7,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,15,0.014314666390419006
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,31,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,31,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,63,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,63,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,127,0.01504533365368843
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,255,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,255,0.01964266722400983
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,511,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,511,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,1023,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,1023,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,2047,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,2047,0.034645333886146545
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,4095,0.0561653325955073
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,4095,0.021503999829292297
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,8191,0.03329066683848699
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,8191,0.09849066535631816
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,16383,0.053077335158983864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,16383,0.1827253301938375
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,32767,0.33826132615407306
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,32767,0.39236267407735187
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,65535,0.7719253698984782
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,1,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,65535,0.7101439634958903
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,3,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,7,0.007194666812817256
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,7,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,15,0.013989333063364029
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,31,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,31,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,63,0.015872000406185787
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,127,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,127,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,255,0.019600000232458115
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,511,0.011946666985750198
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,511,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,1023,0.034645333886146545
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,2047,0.022015998760859173
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,1023,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,2047,0.056320001681645714
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,4095,0.033615998923778534
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,4095,0.0942080020904541
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,8191,0.0536053329706192
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,8191,0.18090667327245077
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,16383,0.30085867643356323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,16383,0.35123201211293537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,32767,0.641706665356954
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,32767,0.7189919948577881
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,1,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,1,0.032287999987602234
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,65535,1.4433333079020183
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,65535,1.2557653586069744
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,3,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,3,0.03330666571855545
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,7,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,7,0.03498666733503342
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,15,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,15,0.04164266586303711
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,31,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,31,0.04164266586303711
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,63,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,63,0.04164266586303711
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,127,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,255,0.030053332448005676
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,255,0.09215466181437175
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,127,0.05222400029500326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,511,0.19950934251149496
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,511,0.17425066232681274
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,1023,0.7024906476338705
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,1023,0.3599199851353963
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,2047,1.4359893798828125
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,2047,0.7297706604003906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,4095,2.8984320958455405
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,4095,1.5650134086608887
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,1,0.013199999928474426
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,8191,3.167231877644857
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,8191,5.733376185099284
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,3,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,7,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,7,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,15,0.0074879998962084455
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,15,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,31,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,31,0.014709333578745524
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,63,0.009445333232482275
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,63,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,127,0.008485333373149237
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,127,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,255,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,511,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,255,0.019658666104078293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,511,0.034645333886146545
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,1023,0.022143999735514324
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,1023,0.05118933320045471
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,2047,0.03172266731659571
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,2047,0.07903466622034709
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,4095,0.14030399918556213
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,8191,0.25668267409006756
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,8191,0.21691733598709106
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,4095,0.047423998514811196
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,16383,0.6640640099843343
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,16383,0.5189973513285319
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,32767,1.4658560752868652
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,32767,1.0511360168457031
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,1,0.017375999440749485
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,3,0.018042666216691334
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,1,0.05208000044027964
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,3,0.053930665055910744
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,65535,1.5880533854166667
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,65535,2.9190826416015625
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,7,0.018543999642133713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,7,0.05684266487757365
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,15,0.022677332162857056
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,15,0.07394666473070781
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,31,0.07389866809050243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,63,0.023552000522613525
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,31,0.023039999107519787
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,63,0.07423999905586243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,127,0.030720000465710957
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,127,0.09284266829490662
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,255,0.19660800695419312
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,255,0.16622933745384216
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,511,0.38741334279378253
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,511,0.33297065893809
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,1023,0.6461439927419027
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,1023,1.3450239499409993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,2047,2.8654934565226235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,2047,1.3373440106709797
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,1,0.032085334261258446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,1,0.1002346674601237
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,4095,2.9775946935017905
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,3,0.03328000009059906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,4095,5.783552169799805
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,7,0.034101332227389015
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,7,0.10803199807802837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,3,0.10427733262379964
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,15,0.04164266586303711
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,15,0.13806933164596558
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,31,0.04181333382924398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,31,0.13821333646774292
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,63,0.1386240025361379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,63,0.04286933441956838
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,127,0.19729065895080566
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,127,0.17407999436060587
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,255,0.3816106716791789
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,255,0.33075199524561566
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,511,0.7516160011291504
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,511,0.620031992594401
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,1,0.059392000238100685
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,1023,1.2054186662038167
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,1023,2.850816090901693
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,1,0.19438934326171875
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,3,0.061610668897628784
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,3,0.2034346659978231
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,7,0.06382933259010315
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,7,0.211626668771108
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,15,0.08004266520341237
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,31,0.08072533210118611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,15,0.2698240081469218
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,31,0.26999467611312866
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,63,0.20053333044052124
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,63,0.2706773281097412
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,127,0.3843413194020589
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,127,0.3556693394978841
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,255,0.7497386932373047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,255,0.6442240079243978
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,511,1.4856534004211426
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,1,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,511,1.2115626335144043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,3,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,3,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,7,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,7,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,15,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,15,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,63,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,63,0.014853333433469137
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,31,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,127,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,127,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,255,0.03346133232116699
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,511,0.021333334346612293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,255,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,511,0.045909335215886436
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,1023,0.026965332527955372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,2047,0.04095999896526337
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,1023,0.06789333124955495
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,2047,0.11281067132949829
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,4095,0.20206934213638306
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,4095,0.21606399615605673
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,8191,0.7028053601582845
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,8191,0.40004265308380127
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,16383,0.8173120021820068
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,16383,1.4518613815307617
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,32767,1.5866293907165527
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,32767,2.921471913655599
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,1,0.11417599519093831
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,65535,3.214336077372233
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,fp8,1,0.37751468022664386
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,65535,5.83628781636556
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,3,0.1186133325099945
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,fp8,3,0.39816534519195557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,7,0.12458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,fp8,7,0.41523198286692303
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,15,0.1585493286450704
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,fp8,15,0.532975991566976
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,31,0.21998933951059976
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,63,0.39509332180023193
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,fp8,31,0.5326506694157919
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,fp8,63,0.5507200161616007
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,127,0.7598079840342203
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,fp8,127,0.6980266571044922
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,255,1.4865066210428874
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,fp8,255,1.2690773010253906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,float16,1,0.22374399503072104
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,fp8,1,0.7500800291697184
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,float16,3,0.233130673567454
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,fp8,3,0.7920640309651693
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,float16,7,0.258730669816335
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,fp8,7,0.8263680140177408
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,float16,15,0.33316266536712646
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,fp8,15,1.07042129834493
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,float16,31,0.4358826478322347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,fp8,31,1.0813439687093098
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,float16,63,0.7830186684926351
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,fp8,63,1.088853359222412
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,1,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,float16,127,1.5090346336364746
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,1,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,3,0.007280000175038974
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,3,0.023039999107519787
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,7,0.007290666922926903
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,fp8,127,1.382570743560791
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,7,0.023232000569502514
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,15,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,15,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,31,0.026799999177455902
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,63,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,63,0.027136000494162243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,127,0.027477333943049114
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,127,0.008885333314538002
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,255,0.010879999647537867
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,255,0.03293866664171219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,511,0.017237332959969837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,511,0.0540533314148585
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,1023,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,1023,0.09762133161226909
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,2047,0.36181334654490155
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,2047,0.18383467197418213
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,4095,0.7195306619008383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,4095,0.3805866638819377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,8191,1.4571520487467449
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,8191,0.7813119888305664
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,16383,2.8670291900634766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,16383,1.5355413754781086
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,1,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,1,0.029696000119050343
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,3,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,3,0.029525332152843475
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,32767,5.803690592447917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,32767,3.214336077372233
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,7,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,7,0.029525332152843475
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,15,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,15,0.029696000119050343
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,31,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,31,0.03139200061559677
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,63,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,127,0.01191466674208641
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,63,0.03737599899371465
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,127,0.037205333511034645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,255,0.037717332442601524
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,255,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,511,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,511,0.045909335215886436
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,1023,0.027978666126728058
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,1023,0.077824001510938
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,2047,0.05171200136343638
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,2047,0.14216533303260803
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,4095,0.48686933517456055
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,4095,0.2711893320083618
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,8191,1.010522683461507
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,8191,0.5599573453267416
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,16383,1.167354663213094
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,16383,2.154154618581136
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,3,0.011600000162919363
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,7,0.00730666642387708
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,15,0.011301333705584208
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,31,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,15,0.007151999821265538
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,31,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,63,0.008010666817426682
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,63,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,127,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,127,0.013776000589132309
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,255,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,255,0.0138026662170887
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,511,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,511,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,1023,0.011546666423479715
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,1023,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,2047,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,2047,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,4095,0.01257066677014033
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,4095,0.01911466692884763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,8191,0.048298666874567665
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,8191,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,16383,0.030207999050617218
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,16383,0.07526400188604991
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,32767,0.04437333345413208
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,32767,0.1360213359196981
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,65535,0.25036799907684326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,65535,0.07509333391984303
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,1,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,3,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,7,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,15,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,31,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,63,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,31,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,63,0.013674666484196981
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,127,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,255,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,255,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,511,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,511,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,1023,0.01179733375708262
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,1023,0.018432000031073887
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,2047,0.012613333761692047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,2047,0.018944000204404194
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,4095,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,4095,0.048298666874567665
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,8191,0.030031998952229817
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,8191,0.07526400188604991
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,16383,0.136053333679835
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,16383,0.04454400142033895
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,32767,0.0747519979874293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,32767,0.24930665890375772
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,1,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,65535,0.5853866736094157
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,1,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,65535,0.4869120121002197
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,3,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,3,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,7,0.00955200009047985
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,7,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,15,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,15,0.023039999107519787
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,31,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,31,0.03141333411137263
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,63,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,63,0.0315733328461647
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,127,0.011946666985750198
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,127,0.031744000812371574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,255,0.03976533313592275
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,255,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,511,0.025770666698614757
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,511,0.06997333467006683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,1023,0.04659200211366018
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,1023,0.13195733229319254
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,2047,0.4884479840596517
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,2047,0.25945067405700684
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,4095,1.0856106281280518
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,4095,0.5509173472722372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,8191,2.1469599405924478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,1,0.007205333560705185
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,8191,1.1818666458129883
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,1,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,3,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,7,0.0074879998962084455
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,7,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,15,0.011946666985750198
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,31,0.007471999774376552
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,63,0.00786666696270307
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,31,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,63,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,127,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,255,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,255,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,511,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,511,0.01791999985774358
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,1023,0.03259733319282532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,1023,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,2047,0.020992000897725422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,2047,0.048298666874567665
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,4095,0.029525332152843475
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,4095,0.07560533285140991
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,8191,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,8191,0.1360213359196981
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,16383,0.07589866717656453
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,16383,0.24934399127960205
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,32767,0.5867520173390707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,32767,0.48264535268147785
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,1,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,65535,1.0525013605753581
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,65535,0.7934292952219645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,1,0.045738667249679565
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,3,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,3,0.03942399968703588
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,7,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,7,0.04126933217048645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,15,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,31,0.017237332959969837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,15,0.04232533276081085
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,31,0.054101333022117615
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,63,0.017407999684413273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,63,0.05410666763782501
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,127,0.01826133330663045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,127,0.05444266895453135
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,255,0.023381332556406658
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,255,0.06845866640408833
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,511,0.043178667624791466
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,511,0.12270933389663696
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,1023,0.29320534070332843
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,1023,0.23366934061050415
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,2047,0.9700693289438883
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,2047,0.4833279848098755
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,1,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,1,0.07543466488520305
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,4095,2.1481812795003257
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,3,0.02402133246262868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,4095,1.011029322942098
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,3,0.07543466488520305
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,7,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,7,0.07918933530648549
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,15,0.025397333006064098
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,15,0.08258133133252461
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,31,0.0315733328461647
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,31,0.10291199882825215
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,63,0.032042667269706726
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,127,0.03310933212439219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,63,0.10308266679445903
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,127,0.10356799761454265
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,255,0.04437333345413208
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,255,0.13107200463612875
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,511,0.2399466633796692
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,511,0.28471465905507404
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,1023,0.5633706649144491
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,1023,0.47086934248606366
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,1,0.04437333345413208
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,1,0.14148267110188803
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,3,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,3,0.14148267110188803
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,7,0.04588800172011057
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,7,0.1486293375492096
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,15,0.04778666794300079
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,15,0.15530666708946228
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,31,0.05973333120346069
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,31,0.20036266247431436
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,63,0.06006399790445963
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,63,0.20053333044052124
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,127,0.07645866771539052
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,127,0.20070399840672812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,255,0.28547199567159015
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,255,0.25924267371495563
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,511,0.5587626695632935
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,511,0.48452266057332355
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,3,0.010970667004585266
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,7,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,31,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,31,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,63,0.008325333396593729
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,63,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,127,0.013658666362365087
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,255,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,511,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,511,0.031231999397277832
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,1023,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,1023,0.04351999859015147
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,2047,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,2047,0.06553600231806438
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,4095,0.03839999934037527
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,4095,0.10905599594116211
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,8191,0.06348800162474315
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,8191,0.1962666710217794
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,16383,0.5061866839726766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,16383,0.3701866865158081
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,32767,1.0516479810078938
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,32767,0.812544027964274
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,65535,2.2181545893351235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,65535,1.2173759937286377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,float16,1,0.08515733480453491
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,float16,3,0.08533333738644917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,fp8,3,0.2751306692759196
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,fp8,1,0.274944007396698
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,float16,7,0.08755200107892354
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,fp8,7,0.2913279930750529
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,float16,15,0.09130666653315227
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,fp8,31,0.39110398292541504
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,float16,63,0.15018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,fp8,15,0.30532266696294147
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,fp8,63,0.3916800022125244
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,float16,127,0.28893866141637164
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,float16,31,0.1153706709543864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,fp8,127,0.40447998046875
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,float16,255,0.562175989151001
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,fp8,255,0.5162186622619629
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,float16,1,0.16571733355522156
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,float16,3,0.16622933745384216
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,fp8,1,0.5464746554692587
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,fp8,3,0.5468159914016724
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,fp8,7,0.5818026860555013
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,float16,7,0.17032533884048462
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,float16,15,0.17919999361038208
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,fp8,15,0.6048426628112793
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,float16,31,0.23756800095240274
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,fp8,31,0.7758506933848063
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,float16,63,0.30088533957799274
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,fp8,63,0.7936000029246012
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,float16,127,0.5713920195897421
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,1,0.006826666494210561
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,1,0.019968000551064808
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,fp8,127,0.7999146779378256
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,3,0.008026666939258575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,7,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,7,0.006826666494210561
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,3,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,15,0.007311999797821045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,15,0.020474666108687718
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,31,0.0074560002734263735
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,63,0.008416000132759413
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,63,0.025429333249727886
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,31,0.021503999829292297
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,127,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,127,0.025600001215934753
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,255,0.008901333436369896
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,255,0.025237334271272022
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,511,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,511,0.03089066594839096
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,1023,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,2047,0.024746666351954143
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,2047,0.09557333588600159
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,1023,0.052239999175071716
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,4095,0.04541866481304169
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,4095,0.18090667327245077
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,8191,0.544426679611206
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,8191,0.3524159987767537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,16383,1.0011306603749592
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,16383,0.7587839762369791
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,1,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,1,0.03242666771014532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,32767,2.20467201868693
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,32767,1.2139519850413005
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,3,0.010373333469033241
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,3,0.03259733319282532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,7,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,7,0.03242666771014532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,15,0.010101333260536194
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,15,0.03427733232577642
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,31,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,31,0.03908266623814901
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,63,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,63,0.03908266623814901
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,127,0.01257066677014033
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,127,0.03957333415746689
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,255,0.015674666812022526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,255,0.04778666794300079
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,511,0.028330666323502857
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,1023,0.051882664362589516
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,511,0.08004266520341237
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,1023,0.14421332875887552
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,2047,0.4867413441340129
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,2047,0.27938133478164673
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,4095,1.005568027496338
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,4095,0.5667839845021566
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,8191,2.1841920216878257
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,8191,1.175920009613037
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,1,0.007365333537260692
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,1,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,16383,2.3309812545776367
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,16383,4.315648078918457
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,3,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,7,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,15,0.013983999689420065
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,31,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,31,0.015216000378131866
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,63,0.015072000523408255
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,127,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,127,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,255,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,255,0.019600000232458115
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,511,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,511,0.020175999651352566
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,1023,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,1023,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,2047,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,2047,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,4095,0.01310933381319046
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,4095,0.05000533163547516
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,8191,0.030378667016824085
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,8191,0.07770666480064392
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,16383,0.04471466441949209
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,16383,0.13944000005722046
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,32767,0.07372800012429555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,32767,0.25481067101160687
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,1,0.0074986666440963745
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,65535,0.5744640032450358
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,1,0.013461332768201828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,65535,0.49817601839701336
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,3,0.007360000163316727
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,3,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,7,0.013130666067202887
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,15,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,31,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,31,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,63,0.008250666782259941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,63,0.014538666854302088
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,127,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,127,0.008687999720374743
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,255,0.00890666681031386
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,255,0.019626667102177937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,511,0.011893333246310553
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,511,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,1023,0.012485332787036896
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,1023,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,4095,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,4095,0.07828266421953838
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,8191,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,2047,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,2047,0.05085866649945577
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,8191,0.13994666934013367
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,16383,0.07333866755167644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,16383,0.25550933678944904
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,32767,0.5760000149408976
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,32767,0.5046613216400146
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,1,0.023045333723227184
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,1,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,3,0.009679999823371569
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,65535,1.1584853331247966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,65535,1.04857603708903
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,3,0.023552000522613525
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,7,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,7,0.024405332903067272
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,15,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,31,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,15,0.032255999743938446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,31,0.032272001107533775
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,63,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,63,0.03242666771014532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,127,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,127,0.040106666584809623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,255,0.04811733464399973
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,255,0.07082666456699371
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,511,0.13329066832860312
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,511,0.04915200173854828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,1023,0.4848639965057373
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,1023,0.26129066944122314
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,2047,0.9973759651184082
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,2047,0.5532799959182739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,4095,2.1746346155802407
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,4095,1.1567786534627278
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,8191,4.290047963460286
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,1,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,8191,2.378063996632894
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,3,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,3,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,7,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,15,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,63,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,31,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,31,0.014511999984582266
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,127,0.015146666516860327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,255,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,255,0.019445333629846573
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,511,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,511,0.034474665919939675
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,1023,0.0506879985332489
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,1023,0.021162666380405426
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,2047,0.029696000119050343
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,2047,0.07816533247629802
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,4095,0.04414399961630503
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,4095,0.14028799533843994
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,8191,0.07423999905586243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,16383,0.5736106634140015
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,8191,0.2563413381576538
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,16383,0.5072213411331177
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,32767,1.1521706581115723
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,32767,1.0477386315663655
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,1,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,65535,1.5665173530578613
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,1,0.04215466479460398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,65535,2.1969920794169107
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,3,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,7,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,7,0.04555733501911163
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,3,0.04369066655635834
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,15,0.017407999684413273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,31,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,15,0.05574933191140493
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,31,0.05580799778302511
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,63,0.018432000031073887
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,63,0.05597866574923197
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,127,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,127,0.06980266670385997
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,255,0.045909335215886436
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,511,0.291157325108846
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,255,0.12441600362459819
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,511,0.23682665824890137
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,1023,0.4865493377049764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,1023,0.9688746929168701
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,2047,2.148138682047526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,2047,1.017514705657959
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,1,0.024746666351954143
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,1,0.07987200220425923
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,4095,4.339200019836426
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,4095,2.2005759874979653
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,3,0.025770666698614757
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,3,0.08311466872692108
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,7,0.026149332523345947
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,7,0.08601599931716919
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,15,0.032085334261258446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,15,0.10598400235176086
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,31,0.03259733319282532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,31,0.10596799850463867
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,63,0.03314133236805598
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,127,0.05563733478387197
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,63,0.10618133346239726
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,255,0.2863786617914836
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,127,0.13344533244768778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,255,0.24473599592844644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,511,0.5633706649144491
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,511,0.47598934173583984
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,1023,2.049194653828939
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,1023,0.9256479740142822
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,1,0.04539733131726583
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,1,0.1474560002485911
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,3,0.1537493367989858
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,7,0.07338666419188182
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,3,0.04710400104522705
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,7,0.16059733430544534
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,15,0.06109866499900818
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,15,0.2053119937578837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,31,0.06144000093142191
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,31,0.20564266045888266
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,63,0.11739200353622437
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,63,0.20599466562271118
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,127,0.29047467311223346
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,127,0.2797226707140605
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,255,0.5621866782506307
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,255,0.4896426598230998
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,1,0.007290666922926903
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,1,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,511,1.108138640721639
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,511,0.9239892959594727
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,3,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,7,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,15,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,31,0.014864000181357065
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,31,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,63,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,127,0.008687999720374743
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,127,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,255,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,255,0.03310933212439219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,511,0.020138667275508244
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,511,0.04571199913819631
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,1023,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,1023,0.06758399804433186
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,2047,0.037717332442601524
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,2047,0.11161067088445027
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,4095,0.06126933296521505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,4095,0.20172800620396933
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,8191,0.5029546817143759
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,8191,0.385535995165507
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,16383,1.020074685414632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,16383,0.8038400014241537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,32767,2.1978185971577964
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,32767,1.563477357228597
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,1,0.08689600229263306
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,65535,2.3980372746785483
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,65535,4.377087910970052
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,fp8,1,0.2853546738624573
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,3,0.08994133273760478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,fp8,3,0.2983253399531047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,7,0.0942080020904541
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,fp8,7,0.3131733338038127
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,15,0.11844266454378764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,fp8,15,0.4015786647796631
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,31,0.1413279970486959
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,63,0.29781333605448407
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,fp8,31,0.40140799681345624
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,fp8,63,0.41608532269795734
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,fp8,127,0.5268906752268473
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,127,0.5713920195897421
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,fp8,255,0.9559040069580078
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,255,1.1129173437754314
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,float16,1,0.16918933391571045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,fp8,1,0.5641013383865356
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,float16,3,0.17527467012405396
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,fp8,3,0.5939199924468994
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,float16,7,0.18636800845464072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,fp8,7,0.6212266683578491
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,float16,15,0.2474666635195414
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,fp8,15,0.7975253264109293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,float16,31,0.3295573393503825
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,fp8,31,0.8111786842346191
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,float16,63,0.589141329129537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,1,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,1,0.02250666668017705
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,fp8,63,0.8198826313018799
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,float16,127,1.13373867670695
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,3,0.007125333572427432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,fp8,127,1.042090654373169
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,3,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,7,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,15,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,7,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,15,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,31,0.0266239990790685
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,31,0.008314666648705801
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,63,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,63,0.02679466704527537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,127,0.00867733359336853
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,127,0.02699200063943863
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,255,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,511,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,255,0.03259733319282532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,511,0.053904001911481224
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,1023,0.02474133421977361
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,1023,0.09676800171534221
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,2047,0.04539733131726583
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,2047,0.18362667163213095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,4095,0.5410133202870687
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,4095,0.359935998916626
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,8191,0.7698773543039957
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,8191,1.0840746561686199
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,16383,2.1599574089050293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,16383,1.5182666778564453
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,32767,4.348586718241374
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,1,0.006810666372378667
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,1,0.020256000260512035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,32767,2.402986685434977
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,3,0.0069440001000960665
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,3,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,7,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,7,0.020165332903464634
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,15,0.02027733375628789
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,31,0.007738666608929634
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,31,0.02189333240191142
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,15,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,63,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,127,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,63,0.025568000972270966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,127,0.025429333249727886
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,255,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,255,0.025600001215934753
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,511,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,511,0.031231999397277832
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,1023,0.017237332959969837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,1023,0.05273599922657013
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,2047,0.029205332199732464
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,2047,0.09540266791979472
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,4095,0.36078933874766034
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,4095,0.18107734123865762
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,8191,0.7301119963328043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,8191,0.37324798107147217
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,16383,1.4370346069335938
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,16383,0.7715840339660645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,1,0.011055999745925268
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,32767,2.9364906946818032
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,32767,1.62116273244222
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,3,0.011253333340088526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,7,0.007653333246707916
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,7,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,15,0.01129066695769628
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,31,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,63,0.008000000069538752
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,31,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,127,0.008309333274761835
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,63,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,127,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,255,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,255,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,511,0.008389333263039589
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,511,0.018090666582187016
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,1023,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,1023,0.020138667275508244
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,2047,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,2047,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,4095,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,4095,0.018650667121013004
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,8191,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,8191,0.018474667022625606
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,16383,0.014842666685581207
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,16383,0.053930665055910744
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,32767,0.03328000009059906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,32767,0.0962559978167216
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,65535,0.053247998158137
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,65535,0.1786880095799764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,131071,0.37990399201711017
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,131071,0.3326293428738912
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,7,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,15,0.007285333548982938
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,31,0.007376000285148621
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,15,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,31,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,63,0.007706666365265846
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,63,0.013503999759753546
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,127,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,127,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,255,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,255,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,511,0.01791999985774358
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,511,0.008527999743819237
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,1023,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,1023,0.018453333526849747
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,2047,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,2047,0.018816000471512478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,4095,0.012762666990359625
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,4095,0.032255999743938446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,8191,0.021333334346612293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,8191,0.054101333022117615
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,16383,0.03309866786003113
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,16383,0.09574400385220845
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,32767,0.052906667192777
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,32767,0.17890665928522745
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,65535,0.37533867359161377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,65535,0.33126399914423627
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,1,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,1,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,131071,0.7818240324656168
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,3,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,131071,0.6927359898885092
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,3,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,7,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,7,0.021221332252025604
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,15,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,31,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,15,0.021333334346612293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,31,0.023552000522613525
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,63,0.013679999858140945
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,63,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,127,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,255,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,255,0.029525332152843475
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,511,0.051039998730023704
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,511,0.0194560003777345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,1023,0.031744000812371574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,1023,0.09454933802286784
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,2047,0.20258132616678873
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,2047,0.18238399426142374
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,4095,0.6900053024291992
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,4095,0.38075733184814453
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,8191,1.451690673828125
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,8191,0.7819946606953939
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,1,0.011477333803971609
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,16383,2.8917547861735025
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,16383,2.2843626340230307
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,3,0.011594666788975397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,7,0.007311999797821045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,15,0.007290666922926903
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,15,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,31,0.007658666620651881
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,63,0.008005333443482717
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,31,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,63,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,127,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,127,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,255,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,255,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,511,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,511,0.017797333498795826
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,1023,0.01179733375708262
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,1023,0.01826133330663045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,2047,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,2047,0.03224000086386999
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,4095,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,4095,0.054101333022117615
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,8191,0.034474665919939675
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,8191,0.09181867043177287
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,16383,0.05649066468079885
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,16383,0.15735466281572977
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,32767,0.3300693432490031
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,32767,0.29337600866953534
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,65535,0.7290879885355631
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,65535,0.6280106703440348
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,1,0.009914666414260864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,1,0.029674666623274486
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,3,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,131071,1.4827466011047363
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,131071,0.8106666405995687
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,3,0.029696000119050343
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,7,0.03345066557327906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,7,0.031061333914597828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,15,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,15,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,31,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,31,0.040448000033696495
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,63,0.013317332913478216
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,127,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,63,0.040448000033696495
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,127,0.040789333482583366
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,255,0.016895999511082966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,511,0.03035199890534083
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,255,0.051029334465662636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,511,0.09079466263453166
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,1023,0.1964906652768453
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,1023,0.17220266660054526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,2047,0.3575466473897298
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,2047,0.6980266571044922
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,4095,0.7229440212249756
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,1,0.0170666662355264
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,4095,1.3527040481567383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,1,0.048810665806134544
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,8191,1.5747413635253906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,8191,2.866186777750651
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,3,0.0170666662355264
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,3,0.04898133377234141
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,7,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,7,0.05119466781616211
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,15,0.018090666582187016
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,15,0.054272000988324486
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,31,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,63,0.02271466702222824
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,31,0.0718506673971812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,63,0.07202133536338806
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,127,0.023552000522613525
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,127,0.0719893326361974
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,255,0.030207999050617218
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,255,0.09079466263453166
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,511,0.19490132729212442
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,511,0.1646880010763804
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,1023,0.3853653271993001
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,1023,0.3283626635869344
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,2047,1.337173302968343
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,1,0.031061333914597828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,2047,0.6399999856948853
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,1,0.09540266791979472
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,3,0.09591466188430786
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,3,0.031231999397277832
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,7,0.032255999743938446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,7,0.10018133123715718
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,15,0.03310933212439219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,15,0.1039573351542155
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,31,0.04067199925581614
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,31,0.13431466619173685
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,63,0.04113066693147024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,63,0.13482667009035745
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,127,0.04266666869322459
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,127,0.13516799608866373
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,255,0.19387733936309814
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,255,0.17117865880330405
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,511,0.377344012260437
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,511,0.32596800724665326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,1,0.007327999919652939
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,1023,0.7470080057779948
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,1023,0.6142293214797974
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,1,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,3,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,7,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,15,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,15,0.011823999385039011
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,31,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,31,0.007360000163316727
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,63,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,63,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,255,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,255,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,511,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,511,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,1023,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,1023,0.032314665615558624
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,2047,0.02218666672706604
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,2047,0.048810665806134544
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,4095,0.07611733178297679
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,4095,0.03258133431275686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,8191,0.0481279989083608
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,8191,0.1360213359196981
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,16383,0.21555199225743613
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,16383,0.25005332628885907
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,32767,0.5109759966532389
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,32767,0.671445369720459
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,65535,1.4771413803100586
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,65535,0.8040106296539307
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,1,0.05819733440876007
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,1,0.1868799924850464
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,131071,2.9603840510050454
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,131071,1.6295253435770671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,3,0.0580266664425532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,3,0.1868799924850464
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,7,0.05973333120346069
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,7,0.19747199614842734
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,15,0.06229333579540253
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,15,0.20599466562271118
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,31,0.07833600044250488
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,63,0.0786240001519521
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,31,0.26316799720128375
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,63,0.26333866516749066
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,127,0.19427732626597086
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,127,0.26385066906611127
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,255,0.3778560161590576
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,255,0.3479893207550049
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,511,0.7417173385620117
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,511,0.6360746622085571
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,fp8,1,0.3657386700312297
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,float16,1,0.11212799946467082
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,float16,3,0.11227200428644817
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,fp8,3,0.3659093379974365
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,float16,7,0.115365336338679
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,fp8,7,0.3877546787261963
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,float16,15,0.12083199620246887
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,float16,31,0.1532586713631948
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,fp8,15,0.40447998046875
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,fp8,31,0.5196799834569296
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,float16,63,0.2044693430264791
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,fp8,63,0.5200213193893433
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,float16,127,0.38178133964538574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,1,0.007365333537260692
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,fp8,127,0.5366186698277792
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,3,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,3,0.01157333329319954
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,float16,255,0.7466560204823812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,7,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,7,0.011221333096424738
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,fp8,255,0.6843732992808024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,15,0.007194666812817256
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,15,0.011087999989589056
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,31,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,31,0.011946666985750198
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,63,0.008352000266313553
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,63,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,127,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,255,0.008858666444818178
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,511,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,255,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,511,0.0315733328461647
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,1023,0.021370666722456615
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,1023,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,2047,0.0262719988822937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,2047,0.06565866867701213
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,4095,0.042805333932240806
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,4095,0.10905599594116211
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,8191,0.2177706758181254
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,8191,0.19643733898798624
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,16383,0.39372801780700684
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,16383,0.7043146292368571
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,32767,1.4774452845255535
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,32767,0.805840015411377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,1,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,65535,2.953045209248861
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,65535,1.6215146382649739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,1,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,3,0.023210667073726654
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,7,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,15,0.007840000092983246
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,15,0.024405332903067272
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,31,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,63,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,31,0.026965332527955372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,63,0.02697066714366277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,127,0.008885333314538002
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,127,0.02716800073782603
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,255,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,255,0.03309866786003113
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,511,0.05444266895453135
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,511,0.0170666662355264
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,1023,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,1023,0.09745066364606221
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,2047,0.3597653309504191
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,2047,0.18363734086354574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,4095,0.7191893259684244
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,4095,0.3800746599833171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,8191,1.4564587275187175
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,8191,0.7814826965332031
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,16383,1.5332694053649902
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,16383,2.8665173848470054
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,1,0.007301333049933116
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,32767,3.2035840352376304
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,1,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,32767,5.801813125610352
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,3,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,7,0.007221333061655362
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,7,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,15,0.013936000565687815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,31,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,63,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,63,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,31,0.015168000012636185
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,127,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,127,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,255,0.008389333263039589
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,255,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,511,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,511,0.020794666061798733
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,1023,0.01227733368674914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,1023,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,2047,0.012432000289360682
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,2047,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,4095,0.01293333371480306
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,4095,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,8191,0.05584000051021576
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,8191,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,16383,0.03310933212439219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,16383,0.09796266754468282
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,32767,0.053120002150535583
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,32767,0.18244266510009766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,65535,0.39097599188486737
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,65535,0.3376053174336751
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,1,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,3,0.007327999919652939
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,131071,0.7736319700876871
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,131071,0.7113386789957682
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,3,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,7,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,15,0.007471999774376552
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,15,0.013978666315476099
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,31,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,31,0.015157333264748255
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,63,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,127,0.015376000354687372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,255,0.0194560003777345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,511,0.011567999919255575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,511,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,1023,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,1023,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,2047,0.012981332838535309
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,2047,0.034304000437259674
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,4095,0.02146133283774058
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,4095,0.05614933371543884
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,8191,0.03310933212439219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,8191,0.09847467144330342
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,16383,0.053077335158983864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,16383,0.18261333306630453
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,32767,0.39099733034769696
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,32767,0.33839468161265057
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,65535,0.7698773543039957
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,65535,0.7080960273742676
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,1,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,131071,1.381205399831136
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,131071,1.5439732869466145
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,1,0.02269333352645238
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,3,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,3,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,7,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,7,0.02421333392461141
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,15,0.01128000020980835
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,15,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,31,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,31,0.025077333052953083
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,63,0.013957332819700241
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,63,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,127,0.03054933249950409
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,255,0.018944000204404194
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,255,0.05220800141493479
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,511,0.0315733328461647
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,511,0.09574400385220845
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,1023,0.202239990234375
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,1023,0.1841119925181071
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,2047,0.705024003982544
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,2047,0.3831466833750407
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,4095,1.4359893798828125
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,4095,0.9634133179982504
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,8191,2.9090134302775064
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,8191,1.5704479217529297
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,1,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,1,0.012986666212479273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,16383,4.5153013865153
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,16383,5.725354512532552
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,3,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,7,0.007189333438873291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,7,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,15,0.007354666789372762
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,15,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,31,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,31,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,63,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,63,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,127,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,127,0.014874666929244995
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,255,0.0194560003777345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,511,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,1023,0.01313599944114685
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,511,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,1023,0.034815999368826546
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,2047,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,2047,0.056320001681645714
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,4095,0.03344533344109853
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,4095,0.09454933802286784
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,8191,0.052895997961362205
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,8191,0.1622719963391622
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,16383,0.34457600116729736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,16383,0.30054400364557904
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,32767,0.719701369603475
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,32767,0.641050656636556
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,65535,1.4429866472880046
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,65535,1.2518400351206462
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,1,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,1,0.03242666771014532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,3,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,131071,2.926080067952474
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,131071,1.5969279607137044
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,3,0.03328000009059906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,7,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,7,0.03498666733503342
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,15,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,15,0.041482667128245033
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,31,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,63,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,31,0.04164266586303711
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,63,0.041797334949175514
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,127,0.017407999684413273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,127,0.05222400029500326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,255,0.030207999050617218
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,255,0.09215999643007915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,511,0.19547200202941895
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,511,0.17407999436060587
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,1023,0.6992533206939697
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,1023,0.359935998916626
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,2047,1.4371840159098308
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,2047,0.7275520165761312
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,4095,2.8973814646402993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,4095,1.5663785934448242
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,1,0.017418666432301205
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,1,0.052383999029795326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,8191,3.1692800521850586
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,8191,5.743786493937175
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,3,0.054101333022117615
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,3,0.018090666582187016
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,7,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,15,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,7,0.056832000613212585
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,15,0.07372800012429555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,31,0.023045333723227184
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,31,0.07372800012429555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,63,0.023552000522613525
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,127,0.03089066594839096
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,127,0.09265599648157756
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,63,0.07406933108965556
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,255,0.19575466712315878
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,255,0.16639999548594156
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,511,0.38690133889516193
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,511,0.3327999909718831
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,1023,1.3424639701843262
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,1023,0.6449493169784546
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,1,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,2047,2.8686240514119468
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,1,0.10018666585286458
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,2047,1.330346663792928
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,3,0.03328000009059906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,3,0.10393599669138591
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,7,0.0341333324710528
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,7,0.10820266604423523
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,15,0.04164266586303711
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,15,0.1384106675783793
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,31,0.04181333382924398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,63,0.043007999658584595
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,31,0.13818132877349854
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,63,0.1384106675783793
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,127,0.19676266113917032
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,127,0.174234668413798
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,255,0.37956265608469647
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,255,0.33075199524561566
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,511,0.7531519730885824
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,511,0.6198773384094238
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,1023,1.205077330271403
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,1,0.01332266628742218
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,1023,2.8454132080078125
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,3,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,7,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,7,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,15,0.014186666657527288
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,31,0.008181333541870117
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,31,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,63,0.008485333373149237
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,127,0.008538666491707167
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,255,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,255,0.019626667102177937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,511,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,511,0.03498666733503342
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,1023,0.022357332209746044
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,2047,0.03107733279466629
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,2047,0.07901866734027863
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,1023,0.05085866649945577
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,4095,0.04743466774622599
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,4095,0.13994666934013367
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,8191,0.21553067366282144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,8191,0.2566986680030823
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,16383,0.6584906578063965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,16383,0.5191520055135092
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,32767,1.4665385882059734
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,32767,1.0523306528727214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,65535,2.9163519541422525
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,65535,1.5880212783813477
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,1,0.059392000238100685
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,1,0.19438934326171875
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,3,0.061610668897628784
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,131071,5.852464040120442
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,131071,3.225088119506836
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,3,0.2034346659978231
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,7,0.06400000055631001
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,7,0.21145600080490112
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,15,0.08002133170763652
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,15,0.26999467611312866
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,31,0.08089600006739299
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,63,0.20053333044052124
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,31,0.26999467611312866
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,63,0.27046932776769
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,127,0.3556693394978841
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,127,0.3843413194020589
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,255,0.7485439777374268
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,255,0.6430720090866089
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,511,1.484458605448405
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,1,0.11449600259462993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,511,1.2112212975819905
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,fp8,1,0.377344012260437
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,3,0.11827199657758077
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,fp8,3,0.39816534519195557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,7,0.12475732962290446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,fp8,7,0.41557331879933673
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,15,0.1590506633122762
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,31,0.22050132354100546
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,fp8,15,0.532480001449585
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,63,0.3942399819691976
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,fp8,63,0.5524479945500692
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,fp8,31,0.5349066654841105
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,1,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,127,0.7587839762369791
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,fp8,127,0.696832021077474
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,1,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,3,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,255,1.4841173489888508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,fp8,255,1.2678826649983723
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,7,0.013424000392357508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,15,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,31,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,31,0.014554666976133982
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,63,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,127,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,127,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,255,0.03345066557327906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,255,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,511,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,1023,0.025941332181294758
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,511,0.04607999821503957
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,1023,0.06796266635258992
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,2047,0.04095999896526337
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,2047,0.11195733149846394
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,4095,0.21504000822703043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,4095,0.20172800620396933
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,8191,0.7145813306172689
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,8191,0.4002133210500081
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,16383,0.8151040077209473
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,16383,1.4539093971252441
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,32767,2.916378657023112
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,32767,1.5846400260925293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,65535,3.2054611841837564
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,65535,5.835776011149089
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,3,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,7,0.0074986666440963745
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,7,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,15,0.007322666545708974
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,15,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,31,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,31,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,63,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,63,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,127,0.00867733359336853
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,127,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,255,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,255,0.013829333086808523
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,511,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,511,0.031221332649389904
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,1023,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,1023,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,2047,0.029525332152843475
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,2047,0.0653653343518575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,4095,0.04215466479460398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,4095,0.10939733187357585
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,8191,0.2172586719195048
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,8191,0.1960960030555725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,16383,0.6970293521881104
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,16383,0.39185067017873126
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,32767,1.4769493738810222
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,32767,0.799402634302775
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,1,0.007162666569153468
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,1,0.01108266661564509
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,3,0.00697066696981589
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,65535,1.6177493731180828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,65535,2.955264091491699
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,7,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,15,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,15,0.011488000551859537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,31,0.00731733317176501
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,31,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,63,0.007834666719039282
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,127,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,63,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,127,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,255,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,255,0.013946666071812311
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,511,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,511,0.018432000031073887
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,1023,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,1023,0.0194560003777345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,2047,0.019626667102177937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,2047,0.012330666184425354
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,4095,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,4095,0.018298666924238205
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,8191,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,8191,0.018432000031073887
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,16383,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,16383,0.01863466699918111
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,32767,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,65535,0.03276800115903219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,32767,0.054101333022117615
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,65535,0.0957973301410675
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,131071,0.054469332098960876
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,131071,0.18056533734003702
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,1,0.011600000162919363
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,3,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,3,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,7,0.007322666545708974
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,7,0.01108266661564509
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,15,0.007141333073377609
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,31,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,15,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,31,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,63,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,63,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,127,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,127,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,255,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,255,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,511,0.008559999987483025
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,511,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,1023,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,1023,0.019968000551064808
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,2047,0.01826133330663045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,2047,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,4095,0.0124746672809124
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,4095,0.018944000204404194
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,8191,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,8191,0.013317332913478216
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,16383,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,16383,0.053930665055910744
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,32767,0.03276800115903219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,32767,0.09608532985051473
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,65535,0.052906667192777
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,65535,0.17919999361038208
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,131071,0.3826346794764201
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,1,0.006826666494210561
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,1,0.020138667275508244
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,131071,0.3322880069414775
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,3,0.006949333474040031
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,3,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,7,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,7,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,15,0.007184000064929326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,31,0.007658666620651881
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,15,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,31,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,63,0.009008000294367472
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,63,0.025146665672461193
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,127,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,127,0.025429333249727886
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,255,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,255,0.025600001215934753
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,511,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,511,0.031744000812371574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,1023,0.017237332959969837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,2047,0.029525332152843475
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,2047,0.09557333588600159
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,1023,0.0524479995171229
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,4095,0.3580586512883504
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,4095,0.18090667327245077
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,8191,0.373418649037679
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,8191,0.7290879885355631
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,16383,1.438037395477295
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,16383,0.7691946824391683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,1,0.007290666922926903
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,1,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,3,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,32767,2.938026746114095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,32767,1.614847977956136
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,7,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,15,0.011429333438475927
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,15,0.007135999699433644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,31,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,31,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,63,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,63,0.01379199946920077
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,127,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,255,0.008341333518425623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,255,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,511,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,511,0.01826133330663045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,1023,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,1023,0.018650667121013004
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,2047,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,2047,0.018432000031073887
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,4095,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,4095,0.03259733319282532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,8191,0.021503999829292297
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,8191,0.053930665055910744
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,16383,0.03259733319282532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,16383,0.09574400385220845
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,32767,0.052906667192777
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,32767,0.1790293256441752
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,65535,0.3772266705830892
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,65535,0.3309226632118225
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,1,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,131071,0.7823359966278076
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,131071,0.6913706461588541
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,1,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,3,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,3,0.020314666132132213
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,7,0.010373333469033241
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,7,0.021333334346612293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,15,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,15,0.021503999829292297
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,31,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,63,0.013818666338920593
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,31,0.023552000522613525
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,63,0.023552000522613525
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,127,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,255,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,255,0.029504001140594482
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,511,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,511,0.051029334465662636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,1023,0.09474666913350423
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,1023,0.032085334261258446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,2047,0.20241065820058188
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,2047,0.1826080083847046
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,4095,0.694271961847941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,4095,0.3824640115102132
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,8191,1.4523733456929524
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,1,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,8191,0.7796160380045573
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,1,0.029717333614826202
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,3,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,16383,2.2865920066833496
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,16383,2.8683945337931314
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,3,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,7,0.03089066594839096
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,7,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,15,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,15,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,31,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,63,0.013151999562978745
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,63,0.040448000033696495
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,31,0.040448000033696495
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,127,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,127,0.040448000033696495
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,255,0.017237332959969837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,255,0.05085866649945577
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,511,0.030378667016824085
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,511,0.09079466263453166
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,1023,0.19508800903956094
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,1023,0.17220266660054526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,2047,0.6869333585103353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,2047,0.3568640152613322
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,1,0.0170666662355264
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,4095,0.7244799931844076
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,4095,1.354581356048584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,1,0.04915200173854828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,3,0.01706133286158244
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,3,0.04915200173854828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,7,0.017557332913080852
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,7,0.051039998730023704
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,15,0.018090666582187016
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,15,0.05461333195368449
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,31,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,31,0.0718506673971812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,63,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,63,0.07167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,127,0.023210667073726654
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,127,0.0721919983625412
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,255,0.030378667016824085
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,255,0.09079466263453166
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,511,0.1945599913597107
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,511,0.16452266772588095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,1023,0.3831466833750407
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,1023,0.3278506596883138
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,1,0.011061333119869232
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,3,0.007157333195209503
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,2047,1.3341013590494792
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,2047,0.6391466856002808
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,3,0.012847999731699625
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,7,0.007157333195209503
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,7,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,15,0.007301333049933116
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,15,0.011946666985750198
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,31,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,31,0.011946666985750198
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,63,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,63,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,255,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,255,0.013690666606028875
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,511,0.008853333070874214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,511,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,1023,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,1023,0.018288000176350277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,2047,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,2047,0.03276800115903219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,4095,0.02197333425283432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,4095,0.053786665201187134
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,8191,0.03362133353948593
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,8191,0.09147733449935913
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,16383,0.05649066468079885
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,16383,0.1576640009880066
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,32767,0.3438933293024699
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,32767,0.2940586606661479
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,65535,0.7284053166707357
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,65535,0.6239573160807291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,1,0.031231999397277832
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,131071,0.807423988978068
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,1,0.09557333588600159
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,131071,1.4829227129618328
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,3,0.031061333914597828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,3,0.09576533238093059
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,7,0.03209600100914637
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,7,0.10017599662144978
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,15,0.03293866664171219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,31,0.040448000033696495
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,15,0.10376532872517903
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,31,0.1346560021241506
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,63,0.04114133367935816
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,63,0.13499733805656433
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,127,0.13499733805656433
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,127,0.04266666869322459
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,255,0.19370667139689127
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,255,0.17117865880330405
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,511,0.3760213454564412
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,511,0.3258026639620463
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,1023,0.7420746485392252
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,1023,0.6135466496149699
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,1,0.0580266664425532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,1,0.1868799924850464
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,3,0.05819733440876007
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,3,0.18705066045125326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,7,0.05973333120346069
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,7,0.19746132691701254
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,15,0.062122667829195656
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,15,0.20616533358891806
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,31,0.07813333471616109
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,31,0.2629866600036621
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,63,0.07884799937407176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,63,0.26366400718688965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,127,0.19473065932591757
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,127,0.26436267296473187
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,1,0.007327999919652939
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,1,0.011109333485364914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,255,0.37768534819285077
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,255,0.34815998872121173
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,3,0.007349333415428798
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,7,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,15,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,15,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,31,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,31,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,511,0.7401813666025797
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,63,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,63,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,511,0.6359039942423502
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,127,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,255,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,255,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,511,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,511,0.01791999985774358
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,1023,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,1023,0.03259733319282532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,2047,0.02236266682545344
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,2047,0.04898133377234141
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,4095,0.03177600105603536
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,4095,0.07594666878382365
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,8191,0.0481333335240682
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,8191,0.1362399955590566
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,16383,0.21794132391611734
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,16383,0.25033066670099896
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,32767,0.5099519888559977
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,32767,0.6714026927947998
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,65535,1.477461338043213
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,65535,0.8033280372619629
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,1,0.013157332936922709
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,3,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,131071,1.624293327331543
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,3,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,131071,2.960554758707682
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,7,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,7,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,15,0.007653333246707916
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,15,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,31,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,63,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,127,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,255,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,255,0.03362133353948593
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,511,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,511,0.04607999821503957
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,1023,0.028005334238211315
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,1023,0.06809600194295247
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,2047,0.04095999896526337
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,2047,0.11400533715883891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,4095,0.2135039965311686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,4095,0.20241065820058188
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,8191,0.6983520189921061
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,8191,0.3993599812189738
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,16383,1.452031930287679
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,16383,0.8134133021036783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,32767,1.5860053698221843
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,32767,2.9231786727905273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,1,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,3,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,65535,5.840895970662435
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,65535,3.2059733072916665
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,7,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,15,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,15,0.0143306665122509
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,31,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,31,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,127,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,127,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,255,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,255,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,511,0.01156266654531161
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,511,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,1023,0.02062400057911873
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,1023,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,2047,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,2047,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,4095,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,8191,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,4095,0.02015999952952067
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,8191,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,16383,0.01469333345691363
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,16383,0.05580799778302511
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,32767,0.03295466552178065
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,32767,0.09898666540781657
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,65535,0.053247998158137
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,65535,0.18278400103251138
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,131071,0.34013867378234863
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,131071,0.39321601390838623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,1,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,3,0.01369599997997284
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,7,0.007141333073377609
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,7,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,15,0.015178666760524115
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,31,0.007967999825874964
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,31,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,63,0.008357333640257517
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,63,0.015376000354687372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,127,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,255,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,255,0.021333334346612293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,511,0.020992000897725422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,511,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,1023,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,1023,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,2047,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,2047,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,4095,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,4095,0.0204373337328434
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,8191,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,8191,0.05580799778302511
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,16383,0.03310933212439219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,16383,0.09859733780225118
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,32767,0.05341866612434387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,32767,0.18243199586868286
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,65535,0.39016000429789227
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,65535,0.3373599847157796
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,1,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,1,0.023210667073726654
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,131071,0.7748266855875651
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,131071,0.7068426609039307
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,3,0.007296000296870868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,3,0.023039999107519787
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,7,0.023039999107519787
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,15,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,15,0.024405332903067272
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,31,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,31,0.026965332527955372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,63,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,63,0.026965332527955372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,127,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,127,0.027306665976842243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,255,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,255,0.03299733251333237
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,511,0.017237332959969837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,511,0.054101333022117615
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,1023,0.028853334486484528
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,1023,0.09830400347709656
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,2047,0.35839998722076416
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,2047,0.18360533316930136
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,4095,0.7195306619008383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,4095,0.37939198811848956
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,8191,1.4569813410441081
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,8191,0.777728001276652
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,16383,1.5315732955932617
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,16383,2.8677120208740234
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,1,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,1,0.013882666826248169
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,3,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,32767,3.2075093587239585
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,3,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,32767,5.795839945475261
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,7,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,15,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,31,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,63,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,31,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,63,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,127,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,127,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,255,0.0194560003777345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,511,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,511,0.02035733312368393
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,1023,0.012655999511480331
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,1023,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,2047,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,2047,0.034304000437259674
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,4095,0.022175999979178112
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,4095,0.05614933371543884
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,8191,0.03310933212439219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,8191,0.09897067149480183
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,16383,0.18295466899871826
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,16383,0.053077335158983864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,32767,0.389631986618042
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,32767,0.3375786542892456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,65535,0.7692106564839681
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,65535,0.7058560053507487
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,1,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,1,0.022357332209746044
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,131071,1.5442186991373699
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,3,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,131071,1.3789653778076172
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,3,0.023029332359631855
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,7,0.010741333166758219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,7,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,15,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,31,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,31,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,63,0.025055999557177227
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,63,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,127,0.015008000036080679
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,127,0.030720000465710957
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,255,0.01911466692884763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,255,0.05205333232879639
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,511,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,511,0.09574400385220845
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,1023,0.202239990234375
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,1023,0.18363734086354574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,2047,0.6971680323282877
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,2047,0.38519465923309326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,4095,1.4365013440450032
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,4095,0.9613653024037679
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,8191,1.574720064798991
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,8191,2.909184137980143
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,1,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,1,0.03257599969704946
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,3,0.010431999961535135
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,16383,5.72760009765625
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,16383,4.521978696187337
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,3,0.03345066557327906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,7,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,7,0.034304000437259674
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,15,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,15,0.04164266586303711
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,31,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,31,0.04147200038035711
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,63,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,63,0.04178666571776072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,127,0.017418666432301205
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,127,0.05205333232879639
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,255,0.030378667016824085
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,255,0.09215999643007915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,511,0.19899733861287436
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,511,0.17390932639439902
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,1023,0.6925653616587321
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,1023,0.35942399501800537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,2047,1.4373547236124675
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,2047,0.7297706604003906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,1,0.01740266631046931
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,4095,2.8979466756184897
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,4095,1.5622827212015789
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,1,0.05205333232879639
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,3,0.01791999985774358
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,3,0.05376000205675761
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,7,0.056703999638557434
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,7,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,15,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,15,0.07423999905586243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,31,0.07372800012429555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,31,0.023045333723227184
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,63,0.023552000522613525
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,63,0.07423999905586243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,127,0.030752000709374745
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,255,0.1960960030555725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,127,0.09284266829490662
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,255,0.16657066345214844
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,511,0.3858773310979207
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,511,0.3323306639989217
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,1023,0.6444533268610636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,1023,1.3383679389953613
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,1,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,3,0.013354666531085968
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,2047,2.8675413131713867
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,2047,1.3300053278605144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,7,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,7,0.007301333049933116
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,15,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,31,0.007877333089709282
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,31,0.014682666709025701
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,127,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,255,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,255,0.01941866676012675
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,511,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,1023,0.013274667163689932
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,511,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,1023,0.034815999368826546
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,2047,0.022677332162857056
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,2047,0.056320001681645714
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,4095,0.03379199902216593
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,4095,0.09494400024414062
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,8191,0.05358933409055074
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,8191,0.16179200013478598
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,16383,0.30105600754419964
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,16383,0.34781865278879803
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,32767,0.7180053393046061
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,32767,0.6391466856002808
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,65535,1.440597375233968
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,65535,1.2521813710530598
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,1,0.032085334261258446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,131071,1.5924906730651855
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,131071,2.9312000274658203
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,3,0.03328000009059906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,1,0.10001066327095032
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,3,0.10392000277837117
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,7,0.0341333324710528
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,7,0.10786133011182149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,15,0.04147200038035711
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,15,0.13822399576505026
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,31,0.04181333382924398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,31,0.13823999961217245
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,63,0.1384106675783793
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,63,0.043178667624791466
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,127,0.17404266198476157
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,127,0.1967786749204
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,255,0.37959468364715576
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,255,0.33058132727940875
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,511,0.7476906776428223
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,511,0.620031992594401
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,1,0.05922133227189382
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,1,0.19453867276509604
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,1023,1.2055892944335938
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,1023,2.833733240763346
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,3,0.06178133189678192
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,3,0.20357332626978555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,7,0.06382933259010315
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,7,0.21211733420689902
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,15,0.07970133423805237
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,15,0.2701653242111206
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,63,0.20053333044052124
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,31,0.27000532547632855
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,31,0.08055466910203297
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,63,0.2706986665725708
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,127,0.3838293155034383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,127,0.3561813433965047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,1,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,255,0.7485439777374268
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,255,0.6444160143534342
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,3,0.007717333113153775
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,3,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,7,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,7,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,15,0.007631999750932057
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,15,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,31,0.007978666573762894
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,31,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,511,1.4755840301513672
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,511,1.2114453315734863
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,63,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,255,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,255,0.019626667102177937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,511,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,511,0.034858666360378265
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,2047,0.031061333914597828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,1023,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,1023,0.051029334465662636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,2047,0.0790293316046397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,4095,0.0473333348830541
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,4095,0.1397760013739268
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,8191,0.21589332818984985
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,8191,0.2570613423983256
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,16383,0.6613333225250244
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,16383,0.5174293518066406
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,32767,1.0490986506144206
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,32767,1.4661973317464192
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,65535,2.922853469848633
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,65535,1.5836159388224285
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,1,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,1,0.08624533812204997
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,3,0.030378667016824085
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,131071,3.218576113382975
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,3,0.08584533135096233
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,131071,5.847040176391602
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,7,0.0315733328461647
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,7,0.08635733524958293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,15,0.032255999743938446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,15,0.09574400385220845
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,31,0.039936001102129616
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,31,0.12526933352152506
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,63,0.040448000033696495
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,63,0.1230506698290507
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,127,0.04181333382924398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,127,0.12800000111262003
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,255,0.05239466826121012
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,255,0.16383999586105347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,511,0.09130666653315227
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,511,0.30481066306432086
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,1023,0.587775985399882
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,1023,0.16554666558901468
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,2047,0.31758399804433185
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,2047,1.1564373175303142
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,4095,0.7050560315450033
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,1,0.030207999050617218
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,1,0.08686932921409607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,4095,2.302805264790853
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,3,0.030207999050617218
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,7,0.0315733328461647
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,3,0.08874666690826416
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,7,0.08789333701133728
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,15,0.032255999743938446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,15,0.09489066402117412
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,31,0.04008000095685323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,31,0.12678933143615723
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,63,0.040618665516376495
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,63,0.1269760032494863
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,127,0.1269760032494863
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,127,0.04181333382924398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,255,0.05478399991989136
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,255,0.16435199975967407
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,511,0.09181867043177287
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,511,0.304639995098114
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,1023,0.16639999548594156
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,1023,0.5867679913838705
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,2047,0.3203413287798564
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,4095,0.7475199699401855
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,2047,1.156266689300537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,4095,2.3057066599527993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,3,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,1,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,3,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,7,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,7,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,15,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,63,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,127,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,255,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,255,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,511,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,511,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,1023,0.03498666733503342
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,1023,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,31,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,4095,0.06656000018119812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,63,0.010362666721145311
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,4095,0.04215466479460398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,2047,0.04915200173854828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,2047,0.025429333249727886
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,1,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,1,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,3,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,7,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,3,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,31,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,7,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,31,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,15,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,15,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,63,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,255,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,511,0.030207999050617218
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,1023,0.03908266623814901
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,511,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,1023,0.02679466704527537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,2047,0.05717333157857259
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,63,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,127,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,127,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,255,0.019626667102177937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,4095,0.0769706666469574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,1,0.010874666273593903
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,3,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,7,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,3,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,7,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,15,0.007344000041484833
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,2047,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,4095,0.2065066695213318
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,31,0.011946666985750198
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,63,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,63,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,127,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,127,0.01368533323208491
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,255,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,255,0.014485333114862442
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,511,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,511,0.030581332743167877
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,1023,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,1023,0.04595200220743815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,2047,0.027482666075229645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,2047,0.0682666649421056
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,4095,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,4095,0.11110400160153706
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,8191,0.0679253339767456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,8191,0.1955839991569519
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,16383,0.1186133325099945
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,16383,0.3708266814549764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,3,0.01110400011142095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,7,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,7,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,15,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,31,0.007829333345095316
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,15,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,31,0.011600000162919363
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,63,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,63,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,127,0.013647999614477158
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,255,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,255,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,511,0.009408000235756239
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,511,0.033157333731651306
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,1023,0.046613335609436035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,1023,0.02120000123977661
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,2047,0.027306665976842243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,2047,0.06489600241184235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,4095,0.1088853379090627
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,4095,0.042650664846102394
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,8191,0.1955839991569519
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,8191,0.06911999980608623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,16383,0.1220266620318095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,16383,0.3700053294499715
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,1,0.007311999797821045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,1,0.010784000158309937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,3,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,7,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,15,0.007653333246707916
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,15,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,31,0.00749333327015241
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,31,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,63,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,127,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,63,0.013466666142145792
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,255,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,255,0.013663999736309052
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,7,0.007482666522264481
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,511,0.009733333562811216
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,3,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,1023,0.021717332303524017
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,2047,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,2047,0.06498133142789204
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,1023,0.045567999283472695
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,4095,0.10905599594116211
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,4095,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,8191,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,8191,0.19592533508936563
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,16383,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,16383,0.36983466148376465
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,3,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,511,0.03306133300065994
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,7,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,7,0.010773333410422007
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,31,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,31,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,63,0.008330666770537695
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,63,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,127,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,255,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,255,0.01516266663869222
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,511,0.009685333197315535
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,511,0.03276800115903219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,1023,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,2047,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,1023,0.04382933179537455
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,2047,0.06553600231806438
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,15,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,4095,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,4095,0.10871466994285583
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,8191,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,8191,0.19575466712315878
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,16383,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,16383,0.36932265758514404
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,1,0.006826666494210561
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,3,0.006826666494210561
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,1,0.01842133328318596
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,3,0.018640000373125076
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,7,0.018432000031073887
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,31,0.00784533346692721
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,15,0.019167999426523846
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,31,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,63,0.02478400121132533
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,63,0.008186666915814081
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,127,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,127,0.02439466615517934
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,255,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,255,0.025941332181294758
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,511,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,511,0.03259733319282532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,1023,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,1023,0.05649066468079885
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,2047,0.029370665550231934
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,2047,0.09419199824333191
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,4095,0.05358933409055074
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,4095,0.2111146648724874
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,8191,0.10291199882825215
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,8191,0.3519146839777629
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,16383,0.2213546633720398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,1,0.006693333387374878
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,1,0.018944000204404194
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,16383,0.6959786415100098
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,3,0.006655999769767125
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,3,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,7,0.007045333584149678
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,7,0.019285333653291065
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,15,0.0194560003777345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,31,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,63,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,31,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,63,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,127,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,127,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,255,0.008672000219424566
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,255,0.025429333249727886
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,511,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,511,0.03362133353948593
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,1023,0.05922133227189382
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,2047,0.029178666571776073
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,1023,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,2047,0.10205866893132527
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,4095,0.05376000205675761
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,4095,0.17937066157658896
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,8191,0.10308266679445903
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,8191,0.3510719935099284
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,16383,0.22340265909830728
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,16383,0.6956640084584554
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,1,0.006805333619316419
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,1,0.0194560003777345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,3,0.00697066696981589
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,3,0.019626667102177937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,15,0.00701333334048589
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,7,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,15,0.018944000204404194
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,31,0.020992000897725422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,31,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,63,0.024282666544119518
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,127,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,127,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,255,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,255,0.024538666009902954
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,511,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,1023,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,511,0.03276800115903219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,1023,0.05120000243186951
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,2047,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,2047,0.09403733412424724
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,4095,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,4095,0.17971199750900269
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,8191,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,8191,0.3519146839777629
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,16383,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,1,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,16383,0.6957973639170328
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,1,0.01907733331123988
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,3,0.006826666494210561
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,3,0.018992000569899876
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,7,0.006655999769767125
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,7,0.020256000260512035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,15,0.019962667177120846
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,15,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,31,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,31,0.020661332954963047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,63,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,63,0.024117333193620045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,127,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,127,0.025258667767047882
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,255,0.008714666590094566
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,511,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,255,0.025600001215934753
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,511,0.03515733281771342
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,1023,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,1023,0.051541333397229515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,2047,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,2047,0.09489066402117412
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,4095,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,8191,0.3510613441467285
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,4095,0.17956799268722534
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,8191,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,16383,0.034474665919939675
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,16383,0.6966933409372965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,1,0.05580799778302511
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,1,0.1687893271446228
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,3,0.05580799778302511
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,3,0.17203199863433838
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,7,0.05836800237496694
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,7,0.178165336449941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,15,0.06075733403364817
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,15,0.18568533658981323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,31,0.07645866771539052
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,31,0.24951465924580893
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,63,0.0773119976123174
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,63,0.24422399202982584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,127,0.07956266899903615
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,255,0.10069333513577779
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,127,0.24883200724919638
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,255,0.32204800844192505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,511,0.17117865880330405
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,511,0.6022826830546061
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,1023,0.3184640010197957
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,1023,1.1617279847462971
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,2047,0.6183253526687622
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,1,0.056320001681645714
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,1,0.16914665699005127
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,3,0.05649066468079885
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,2047,2.293248017628988
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,3,0.16947199900945029
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,7,0.05853866537412008
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,7,0.17851734161376953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,15,0.06092800199985504
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,31,0.07660800218582153
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,31,0.2505386670430501
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,15,0.1867093245188395
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,63,0.0773119976123174
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,63,0.2486613392829895
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,127,0.08004266520341237
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,127,0.2488266626993815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,255,0.10546666383743286
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,255,0.32255999247233075
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,511,0.17800533771514893
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,1023,0.33075199524561566
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,1023,1.163434664408366
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,511,0.6021120150883993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,2047,0.6285653511683146
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,1,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,2047,2.2918826738993325
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,1,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,3,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,3,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,7,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,15,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,7,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,15,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,31,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,31,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,63,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,63,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,127,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,255,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,255,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,511,0.03566933423280716
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,511,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,1023,0.04863999783992767
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,1023,0.02918400118748347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,2047,0.06775466601053874
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,2047,0.04642133414745331
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,1,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,1,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,3,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,3,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,7,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,7,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,15,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,15,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,31,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,31,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,63,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,63,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,127,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,127,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,255,0.0315733328461647
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,255,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,511,0.04113066693147024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,511,0.03310933212439219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,1023,0.0580266664425532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,1023,0.04795733094215393
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,2047,0.2106026609738668
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,1,0.009759999811649323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,2047,0.0820906658967336
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,1,0.018432000031073887
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,3,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,3,0.018288000176350277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,7,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,7,0.01913600042462349
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,15,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,15,0.019626667102177937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,31,0.022015998760859173
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,31,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,127,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,63,0.022357332209746044
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,63,0.01381333296497663
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,127,0.02218666672706604
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,255,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,255,0.02848000079393387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,511,0.018432000031073887
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,511,0.055125330885251365
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,1023,0.03498666733503342
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,1023,0.09267200032869975
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,2047,0.05378133555253347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,2047,0.19131733973821005
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,4095,0.09985599915186565
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,4095,0.35652267932891846
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,8191,0.19643733898798624
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,8191,0.7156053384145101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,16383,0.42854400475819904
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,1,0.01926400015751521
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,1,0.010079999764760336
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,16383,2.1319680213928223
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,3,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,3,0.018944000204404194
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,7,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,7,0.019626667102177937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,15,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,15,0.019968000551064808
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,31,0.023002666731675465
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,63,0.022885332504908245
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,127,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,63,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,31,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,127,0.023039999107519787
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,255,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,255,0.028501334289709728
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,511,0.049973333875338234
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,511,0.01826133330663045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,1023,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,1023,0.10666666428248088
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,2047,0.05461333195368449
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,2047,0.18141865730285645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,4095,0.10154666503270467
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,4095,0.3561813433965047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,8191,0.20258132616678873
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,8191,0.7157759666442871
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,16383,0.4393119812011719
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,1,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,1,0.018944000204404194
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,16383,2.136576016743978
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,3,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,3,0.019167999426523846
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,7,0.010181333248813948
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,15,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,7,0.019626667102177937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,31,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,15,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,31,0.023039999107519787
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,63,0.013477332890033722
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,63,0.023039999107519787
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,127,0.013616000612576803
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,127,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,255,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,255,0.03242666771014532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,511,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,511,0.0580266664425532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,1023,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,1023,0.09320533275604248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,2047,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,2047,0.18107734123865762
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,4095,0.018944000204404194
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,4095,0.35703468322753906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,8191,0.7162880102793375
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,8191,0.028501334289709728
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,16383,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,1,0.009904000287254652
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,1,0.01911466692884763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,3,0.009957333405812582
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,16383,2.1369172732035318
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,3,0.019253333409627277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,7,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,7,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,15,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,15,0.020138667275508244
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,31,0.011087999989589056
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,31,0.02290133386850357
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,63,0.01349866638580958
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,63,0.023552000522613525
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,127,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,127,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,255,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,255,0.02867199977238973
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,511,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,511,0.05017599960168203
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,1023,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,1023,0.09351467092831929
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,2047,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,2047,0.18107734123865762
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,4095,0.025600001215934753
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,4095,0.3566933472951253
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,8191,0.03737599899371465
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,8191,0.7159466743469238
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,16383,0.058880001306533813
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,1,0.10752000411351521
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,16383,2.1415252685546875
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,3,0.10752000411351521
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,1,0.3304266730944316
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,3,0.33074132601420086
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,7,0.11281067132949829
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,7,0.34884266058603924
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,15,0.11788800358772278
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,31,0.14967466394106546
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,15,0.3701759974161784
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,31,0.48981332778930664
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,63,0.15069866180419922
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,63,0.4910080035527547
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,127,0.155648003021876
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,255,0.19490132729212442
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,127,0.49164799849192303
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,255,0.6384640137354533
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,511,0.33826132615407306
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,511,1.1946667035420735
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,1023,0.6362186670303345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,1,0.1088853379090627
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,1023,2.3145813941955566
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,1,0.33190399408340454
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,3,0.1088853379090627
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,3,0.33177600304285687
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,7,0.11249066392580669
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,7,0.35022934277852374
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,15,0.3671040137608846
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,15,0.1181013286113739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,31,0.1500320037206014
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,31,0.49032533168792725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,63,0.15069866180419922
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,63,0.49083733558654785
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,127,0.15598932902018228
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,127,0.49181334177652997
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,255,0.20480000972747803
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,255,0.6391466856002808
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,511,0.3485013246536255
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,511,1.1950080394744873
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,1023,0.6345386505126953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,1,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,1023,2.3127039273579917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,1,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,3,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,7,0.022357332209746044
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,3,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,7,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,15,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,15,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,31,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,31,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,63,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,63,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,127,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,127,0.025429333249727886
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,255,0.03839999934037527
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,255,0.026965332527955372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,511,0.05120000243186951
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,511,0.038058665891488395
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,1023,0.06894933183987935
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,1023,0.05444266895453135
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,1,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,1,0.031061333914597828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,3,0.03089066594839096
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,3,0.03089066594839096
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,7,0.03089066594839096
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,7,0.031061333914597828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,15,0.03089066594839096
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,15,0.031061333914597828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,31,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,63,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,63,0.031061333914597828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,127,0.03089066594839096
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,31,0.03089066594839096
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,127,0.038058665891488395
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,255,0.03822933385769526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,255,0.04625066618124644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,511,0.054272000988324486
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,511,0.06263466676076253
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,1023,0.2111146648724874
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,1023,0.09045333663622539
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,fp8,1,0.6560426553090414
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,float16,3,0.2111146648724874
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,float16,1,0.210591991742452
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,fp8,3,0.6571146647135416
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,float16,7,0.22101332743962607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,fp8,7,0.6963199774424235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,float16,15,0.23227733373641968
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,fp8,15,0.724992036819458
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,float16,31,0.2954240043958028
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,fp8,31,0.9738240242004395
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,float16,63,0.29730133215586346
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,fp8,63,0.9750186602274576
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,float16,127,0.30702932675679523
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,fp8,127,0.9763519763946533
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,float16,255,0.38331735134124756
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,fp8,255,1.2678986390431721
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,float16,1,0.2136746644973755
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,fp8,1,0.6609919865926107
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,fp8,3,0.6611626545588175
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,float16,3,0.2136746644973755
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,float16,7,0.2208426594734192
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,fp8,7,0.6997546354929606
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,float16,15,0.23244800170262656
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,fp8,15,0.7309172948201498
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,float16,31,0.2955946723620097
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,fp8,31,0.9743359883626302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,float16,63,0.29781333605448407
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,float16,127,0.30822400252024335
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,fp8,63,0.974506696065267
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,float16,255,0.4041386842727661
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,fp8,127,0.9767253398895264
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,fp8,255,1.2687359650929768
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,fp8,1,0.04147200038035711
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,float16,1,0.04113066693147024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,float16,3,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,fp8,3,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,float16,15,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,fp8,7,0.041984001795450844
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,fp8,15,0.04113066693147024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,float16,31,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,fp8,31,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,float16,63,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,fp8,63,0.07048533360163371
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,float16,127,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,fp8,127,0.04334933559099833
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,float16,255,0.04898133377234141
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,fp8,255,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,float16,1,0.05444266895453135
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,fp8,1,0.0554666668176651
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,float16,3,0.053930665055910744
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,float16,7,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,fp8,3,0.05495466788609823
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,fp8,7,0.055125330885251365
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,float16,7,0.05358933409055074
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,float16,15,0.05461333195368449
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,fp8,15,0.05529599885145823
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,float16,31,0.05358933409055074
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,fp8,31,0.05461333195368449
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,float16,63,0.05358933409055074
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,fp8,63,0.05461333195368449
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,float16,127,0.05444266895453135
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,fp8,127,0.06946133573849995
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,float16,255,0.06997333467006683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,fp8,255,0.08533333738644917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,float16,1,0.4169386625289917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,fp8,1,1.3375147183736165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,float16,3,0.41659732659657794
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,float16,7,0.43724799156188965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,float16,15,0.4606399933497111
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,fp8,3,1.3093547026316326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,fp8,7,1.394858678181966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,fp8,15,1.4452053705851238
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,float16,31,0.5879520177841187
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,fp8,31,1.9408213297526042
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,float16,127,0.6106453339258829
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,fp8,63,1.940992037455241
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,float16,1,0.4242773453394572
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,float16,63,0.5908960103988647
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,fp8,1,1.3175466855367024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,fp8,127,1.9459412892659504
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,float16,3,0.4242773453394572
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,fp8,3,1.3189120292663574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,float16,7,0.43676265080769855
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,float16,15,0.4608000119527181
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,fp8,7,1.3991252581278484
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,float16,31,0.5876053174336752
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,fp8,15,1.4624427159627278
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,fp8,31,1.94048007329305
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,float16,63,0.5920266707738241
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,fp8,63,1.9432106018066406
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,float16,127,0.6128640174865723
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,float16,1,0.07389866809050243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,fp8,1,0.07492266595363617
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,fp8,127,1.9473066329956055
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,fp8,3,0.09727999567985535
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,float16,7,0.07389866809050243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,float16,3,0.07406933108965556
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,fp8,7,0.07441066702206929
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,float16,15,0.07389866809050243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,fp8,15,0.07492266595363617
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,float16,31,0.07406933108965556
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,fp8,31,0.07492266595363617
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,fp8,63,0.07458133498827617
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,float16,63,0.07372800012429555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,float16,127,0.07492266595363617
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,float16,1,0.10154666503270467
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,fp8,127,0.077824001510938
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,fp8,1,0.10342400272687276
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,float16,3,0.10171733299891154
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,fp8,3,0.10240000486373901
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,float16,7,0.10154666503270467
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,fp8,7,0.10376532872517903
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,float16,15,0.10171733299891154
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,fp8,15,0.10376532872517903
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,fp8,31,0.10308266679445903
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,float16,31,0.10171733299891154
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,float16,63,0.10154666503270467
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,fp8,63,0.10222933689753215
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,float16,127,0.10291199882825215
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,fp8,127,0.13141333063443503
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,1,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,1,0.02762666592995326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,3,0.009765333185593287
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,3,0.027471999327341717
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,7,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,15,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,7,0.02867199977238973
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,15,0.02935466667016347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,31,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,31,0.03807466725508372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,63,0.012997332960367203
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,63,0.03822933385769526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,127,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,127,0.03839999934037527
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,255,0.016410666207472484
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,255,0.048469334840774536
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,511,0.08874666690826416
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,511,0.027477333943049114
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,1023,0.04690133531888326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,1023,0.169813334941864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,2047,0.08482133348782857
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,2047,0.33348266283671063
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,4095,0.17425066232681274
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,4095,0.6562133232752482
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,8191,0.34918399651845294
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,16383,1.0586453278859456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,8191,1.4160213470458984
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,1,0.009882666791478792
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,1,0.02734400083621343
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,3,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,3,0.027477333943049114
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,16383,4.251477241516113
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,7,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,7,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,31,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,15,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,31,0.03788800040880839
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,15,0.010549332946538925
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,63,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,63,0.038058665891488395
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,127,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,127,0.038058665891488395
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,255,0.016895999511082966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,255,0.04898133377234141
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,511,0.027477333943049114
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,1023,0.04625066618124644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,1023,0.16964266697565714
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,511,0.08840533097585042
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,2047,0.08516266942024231
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,2047,0.3317813277244568
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,4095,0.1838080088297526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,4095,0.6570666631062826
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,8191,0.3715413411458333
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,8191,1.4151679674784343
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,16383,1.07314133644104
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,1,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,1,0.027530667682488758
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,16383,4.253354708353679
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,3,0.00984533317387104
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,3,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,7,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,7,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,15,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,15,0.030207999050617218
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,31,0.012847999731699625
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,31,0.036864000062147774
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,63,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,63,0.03737599899371465
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,127,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,127,0.03822933385769526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,255,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,255,0.04863999783992767
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,511,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,1023,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,1023,0.17032533884048462
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,511,0.0885759989420573
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,2047,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,2047,0.33211733897527057
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,4095,0.02918400118748347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,4095,0.6572373310724894
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,8191,0.04454400142033895
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,8191,1.41704527537028
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,16383,0.06946133573849995
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,3,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,16383,4.258933385213216
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,15,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,31,0.00749333327015241
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,63,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,63,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,127,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,255,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,511,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,511,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,1023,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,2047,0.02679466704527537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,1023,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,2047,0.024746666351954143
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,4095,0.04113066693147024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,4095,0.04113066693147024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,8191,0.07406933108965556
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,8191,0.0576853354771932
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,16383,0.20599466562271118
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,16383,0.13943466544151306
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,float16,1,0.8267093499501547
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,float16,3,0.8280746936798096
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,fp8,1,2.615818659464518
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,fp8,3,2.6144426663716636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,float16,7,0.8702293237050375
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,float16,15,0.9178453286488851
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,fp8,7,2.791237195332845
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,fp8,15,2.8881759643554688
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,float16,31,1.1707733472188313
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,float16,63,1.1797653039296467
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,fp8,31,3.8736213048299155
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,float16,1,0.8459946314493815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,fp8,63,3.876522699991862
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,float16,3,0.8432640234629313
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,fp8,1,2.632362683614095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,fp8,3,2.6316800117492676
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,float16,7,0.869376023610433
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,fp8,7,2.8043947219848633
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,float16,15,0.9178453286488851
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,fp8,15,2.9368321100870767
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,float16,31,1.1719679832458496
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,float16,63,1.1798186302185059
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,fp8,31,3.8746506373087564
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,fp8,63,3.877237319946289
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,fp8,1,0.14045866330464682
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,float16,1,0.13858133554458618
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,float16,3,0.13960533340771994
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,fp8,3,0.14011733730634054
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,float16,7,0.13960533340771994
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,fp8,7,0.14131200313568115
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,float16,15,0.13909332950909933
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,fp8,15,0.14097066720326742
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,float16,31,0.13943466544151306
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,float16,63,0.13960533340771994
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,fp8,31,0.13943466544151306
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,fp8,63,0.14011733730634054
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,float16,1,0.19882667064666748
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,fp8,1,0.2001919945081075
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,float16,3,0.19814399878184
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,fp8,3,0.20053333044052124
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,float16,7,0.19882667064666748
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,fp8,7,0.2001919945081075
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,fp8,15,0.20053333044052124
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,float16,15,0.19814399878184
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,fp8,31,0.2001919945081075
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,float16,31,0.19882667064666748
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,float16,63,0.22971733411153158
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,fp8,63,0.19985065857569376
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,float16,1,1.6476586659749348
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,float16,3,1.6471039454142253
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,float16,7,1.7361920674641926
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,fp8,1,5.228885332743327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,float16,15,1.8300587336222331
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,fp8,3,5.22763729095459
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,fp8,7,5.587088267008464
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,fp8,15,5.777621587117513
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,float16,31,2.341887950897217
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,float16,1,1.6892587343851726
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,float16,3,1.683626651763916
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,fp8,1,5.26095453898112
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,float16,7,1.7344853083292644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,fp8,31,7.741952260335286
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,fp8,3,5.2597761154174805
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,fp8,7,5.621589024861653
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,float16,15,1.8315946261088054
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,fp8,15,5.901653289794922
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,float16,31,2.33949867884318
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,float16,1,0.30105600754419964
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,fp8,1,0.29713066418965656
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,float16,3,0.30668799082438153
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,fp8,31,7.744314829508464
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,fp8,3,0.3027626673380534
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,float16,7,0.3128319978713989
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,fp8,7,0.3092479904492696
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,float16,15,0.31829333305358887
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,fp8,15,0.3145386576652527
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,float16,31,0.32767999172210693
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,fp8,31,0.3295573393503825
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,float16,1,0.42956801255544025
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,fp8,1,0.442197322845459
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,float16,3,0.4341760079065959
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,fp8,3,0.4490240017573039
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,float16,7,0.44441600640614826
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,fp8,7,0.45977600415547687
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,float16,15,0.4522666533788045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,1,0.016751999656359356
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,float16,31,0.4522666533788045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,fp8,15,0.48366932074228924
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,fp8,31,0.5259946584701538
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,1,0.044549331068992615
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,3,0.016864000509182613
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,3,0.04471466441949209
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,7,0.017237332959969837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,7,0.04708800216515859
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,15,0.01791999985774358
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,15,0.04966400067011515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,31,0.02201066662867864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,31,0.06809600194295247
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,63,0.02218666672706604
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,63,0.06825600067774455
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,127,0.06860800087451935
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,127,0.02288000037272771
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,255,0.029535998900731403
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,511,0.048613334695498146
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,255,0.08686932921409607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,511,0.1604266663392385
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,1023,0.08482133348782857
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,1023,0.30959999561309814
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,2047,0.1616213321685791
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,2047,0.6068906784057617
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,4095,0.3421866496404012
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,4095,1.2061013380686443
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,8191,0.7298293113708496
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,1,0.016682667036851246
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,8191,2.6480639775594077
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,1,0.045040001471837364
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,3,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,3,0.045226668318112694
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,7,0.017237332959969837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,7,0.04714133342107137
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,15,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,15,0.017797333498795826
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,31,0.02218666672706604
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,31,0.06775466601053874
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,63,0.022304000953833263
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,63,0.06809600194295247
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,127,0.023039999107519787
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,127,0.06775466601053874
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,511,0.049322664737701416
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,255,0.030207999050617218
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,511,0.16076266765594482
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,255,0.08686932921409607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,1023,0.08738133311271667
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,1023,0.30873600641886395
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,2047,0.16059733430544534
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,2047,0.6070613463719686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,4095,0.36248000462849933
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,4095,1.2064426740010579
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,8191,0.7632213433583578
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,1,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,3,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,8191,2.6472105979919434
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,7,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,15,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,31,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,31,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,63,0.008474666625261307
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,127,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,255,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,255,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,511,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,511,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,1023,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,1023,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,2047,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,2047,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,4095,0.04898133377234141
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,4095,0.04164266586303711
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,8191,0.0682666649421056
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,8191,0.07406933108965556
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,1,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,1,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,3,0.008016000191370646
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,3,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,7,0.008341333518425623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,15,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,31,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,15,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,31,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,63,0.008320000022649765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,63,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,255,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,255,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,511,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,511,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,1023,0.029525332152843475
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,1023,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,2047,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,2047,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,4095,0.056661332647005715
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,4095,0.04113066693147024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,8191,0.2065066695213318
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,1,0.031061333914597828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,1,0.08806399504343669
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,8191,0.07406933108965556
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,3,0.03242666771014532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,3,0.09284266829490662
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,7,0.033285332222779594
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,7,0.09710933764775594
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,15,0.04095999896526337
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,15,0.1281706690788269
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,31,0.04113066693147024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,31,0.1293653349081675
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,63,0.04215466479460398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,63,0.13192533453305563
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,127,0.053077335158983864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,127,0.16759467124938965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,255,0.09026666482289632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,255,0.3068586587905884
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,511,0.16605866948763529
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,511,0.5909813245137533
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,1023,0.31437865893046063
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,1023,1.1641173362731934
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,2047,0.6304426590601603
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,2047,2.314469337463379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,4095,1.4846293131510417
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,1,0.031231999397277832
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,1,0.0885759989420573
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,4095,4.613973299662272
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,3,0.032405334214369454
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,3,0.09318400422732036
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,7,0.03328000009059906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,7,0.09622400005658467
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,15,0.04095999896526337
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,15,0.12902399897575378
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,31,0.04095999896526337
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,31,0.1288533310095469
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,63,0.04232533276081085
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,63,0.13191999991734824
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,127,0.05563733478387197
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,127,0.16605866948763529
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,255,0.09318400422732036
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,255,0.30702932675679523
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,511,0.5911893447240194
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,511,0.17148800690968832
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,1023,0.3282346725463867
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,1023,1.1637760003407795
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,2047,0.6597973505655924
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,2047,2.3145813941955566
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,4095,1.5307092666625977
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,4095,4.614485422770183
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,1,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,1,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,3,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,3,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,7,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,7,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,15,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,15,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,31,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,63,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,31,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,63,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,127,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,127,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,255,0.018090666582187016
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,255,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,511,0.027477333943049114
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,1023,0.026965332527955372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,2047,0.06587733328342438
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,2047,0.04437333345413208
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,4095,0.23398399353027344
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,4095,0.07748266557852428
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,511,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,1023,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,1,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,1,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,15,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,3,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,3,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,7,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,7,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,31,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,31,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,63,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,15,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,127,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,63,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,255,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,255,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,511,0.03788800040880839
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,511,0.027818667391935985
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,1023,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,2047,0.2249386707941691
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,2047,0.07901866734027863
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,4095,0.43485867977142334
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,4095,0.2208426594734192
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,1023,0.05376000205675761
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,1,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,3,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,3,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,7,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,7,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,15,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,31,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,15,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,63,0.008853333070874214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,31,0.014463999619086584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,127,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,127,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,255,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,255,0.03328000009059906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,511,0.04710400104522705
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,511,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,1023,0.02682666728893916
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,1023,0.07082666456699371
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,2047,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,2047,0.11534933249155681
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,4095,0.06673066814740498
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,4095,0.19868266582489014
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,8191,0.11724266409873962
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,16383,0.22062400976816812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,8191,0.37614933649698895
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,16383,0.7301119963328043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,1,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,1,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,3,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,3,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,7,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,7,0.012298667182525
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,15,0.007877333089709282
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,15,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,31,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,31,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,63,0.008853333070874214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,127,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,255,0.009594666461149851
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,255,0.034304000437259674
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,511,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,511,0.04607999821503957
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,1023,0.027999999622503918
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,1023,0.07099733253320058
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,2047,0.040106666584809623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,2047,0.11025066177050273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,4095,0.06638933221499126
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,4095,0.19933867454528809
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,8191,0.11808533469835918
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,8191,0.37649067242940265
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,16383,0.2187946637471517
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,16383,0.7290933132171631
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,1,0.012485332787036896
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,3,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,15,0.007674666742483775
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,15,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,31,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,31,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,63,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,63,0.01469333345691363
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,127,0.00903466654320558
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,3,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,7,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,127,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,7,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,255,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,255,0.03259733319282532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,511,0.04915200173854828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,1023,0.027818667391935985
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,1023,0.067071999112765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,2047,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,2047,0.1114453375339508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,4095,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,4095,0.19985065857569376
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,8191,0.019626667102177937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,16383,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,8191,0.37698133786519367
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,1,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,16383,0.7297706604003906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,3,0.0074560002734263735
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,511,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,3,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,7,0.007674666742483775
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,15,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,7,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,31,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,31,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,63,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,63,0.014698666830857595
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,127,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,127,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,511,0.045226668318112694
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,511,0.021503999829292297
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,1023,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,255,0.03379199902216593
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,255,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,1023,0.06690133114655812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,2047,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,2047,0.1109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,4095,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,4095,0.20070399840672812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,8191,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,8191,0.37717334429423016
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,16383,0.027306665976842243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,16383,0.730282704035441
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,1,0.019285333653291065
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,3,0.007354666789372762
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,3,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,7,0.007344000041484833
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,15,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,7,0.019797333826621372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,15,0.010224000240365664
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,31,0.008346666892369589
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,31,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,63,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,63,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,127,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,127,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,255,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,511,0.017407999684413273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,255,0.03362133353948593
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,511,0.051882664362589516
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,1023,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,1023,0.10410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,2047,0.05273599922657013
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,2047,0.18090667327245077
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,4095,0.35396798451741535
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,4095,0.10001066327095032
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,8191,0.20070399840672812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,8191,0.6987093289693197
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,16383,0.43212799231211346
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,1,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,1,0.020634666085243225
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,16383,1.3853012720743816
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,3,0.020992000897725422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,7,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,7,0.007173333317041397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,15,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,15,0.02218666672706604
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,31,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,31,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,127,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,63,0.025770666698614757
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,127,0.02679466704527537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,255,0.0106133334338665
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,255,0.034815999368826546
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,511,0.017386666188637417
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,511,0.056832000613212585
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,1023,0.028890666862328846
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,1023,0.09489066402117412
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,2047,0.053247998158137
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,2047,0.18107734123865762
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,4095,0.10257066289583842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,4095,0.3541333278020223
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,8191,0.20055999358495077
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,8191,0.6981973648071289
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,16383,0.4386133352915446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,16383,1.3858133951822917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,1,0.006826666494210561
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,1,0.020992000897725422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,3,0.021850667893886566
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,7,0.02080533280968666
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,15,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,31,0.02611200014750163
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,31,0.008389333263039589
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,63,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,63,0.026634665826956432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,127,0.00919999989370505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,127,0.0266239990790685
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,255,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,255,0.031583999594052635
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,511,0.017077332983414333
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,511,0.05239466826121012
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,1023,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,1023,0.09574400385220845
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,15,0.02218666672706604
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,2047,0.18141865730285645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,2047,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,4095,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,4095,0.3543093204498291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,8191,0.027306665976842243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,16383,0.034474665919939675
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,8191,0.6985493501027426
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,1,0.006949333474040031
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,1,0.021333334346612293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,16383,1.3863253593444824
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,7,0.021162666380405426
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,15,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,3,0.022015998760859173
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,15,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,31,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,31,0.025429333249727886
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,63,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,63,0.025429333249727886
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,127,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,127,0.026965332527955372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,255,0.031248000760873158
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,511,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,511,0.05273599922657013
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,1023,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,255,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,1023,0.09524266918500264
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,2047,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,4095,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,2047,0.18190399805704752
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,8191,0.03089066594839096
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,4095,0.35447466373443604
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,8191,0.6985599994659424
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,16383,0.04539733131726583
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,1,0.05734399954477946
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,16383,1.385642687479655
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,1,0.17122133572896323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,3,0.06041066845258077
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,3,0.17971199750900269
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,7,0.062463998794555664
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,7,0.1879040002822876
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,15,0.07833600044250488
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,31,0.07834133505821228
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,15,0.25190399090449017
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,31,0.25190399090449017
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,63,0.25190399090449017
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,63,0.08038400113582611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,127,0.3252906600634257
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,127,0.10001066327095032
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,255,0.1699626644452413
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,255,0.6055253346761068
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,511,0.3213653365770976
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,1023,0.6191786527633667
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,511,1.1698026657104492
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,2047,1.2356266975402832
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,1023,2.3094612757364907
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,1,0.0576853354771932
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,2047,4.5964053471883135
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,1,0.1718613306681315
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,3,0.06007466713587443
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,3,0.18056533734003702
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,7,0.06248533229033152
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,7,0.18978132804234824
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,15,0.07867200175921123
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,15,0.25224532683690387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,31,0.252074658870697
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,31,0.07867733140786488
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,63,0.08089600006739299
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,63,0.25224532683690387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,127,0.10568533341089885
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,127,0.3258026639620463
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,255,0.17612799008687338
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,511,0.33075199524561566
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,255,0.6058666706085205
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,1023,0.6488746802012125
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,511,1.170090675354004
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,2047,1.3096960385640461
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,1023,2.3089493115743003
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,1,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,1,0.016895999511082966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,2047,4.596565246582031
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,3,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,3,0.016895999511082966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,7,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,15,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,7,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,15,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,31,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,31,0.017407999684413273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,63,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,127,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,127,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,255,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,511,0.04539733131726583
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,255,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,511,0.03242666771014532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,1023,0.04949333270390829
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,1023,0.06502399841944377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,2047,0.23825067281723022
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,2047,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,1,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,1,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,3,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,3,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,7,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,7,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,15,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,15,0.022015998760859173
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,31,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,31,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,63,0.019626667102177937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,63,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,127,0.023210667073726654
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,127,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,255,0.03925333420435587
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,255,0.0314026673634847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,511,0.056320001681645714
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,511,0.0506879985332489
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,1023,0.22630399465560913
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,2047,0.4374186595280965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,1,0.010122666756312052
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,2047,0.22681599855422974
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,1,0.019274666905403137
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,3,0.010384000216921171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,3,0.020138667275508244
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,7,0.010405333091815313
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,7,0.019797333826621372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,15,0.02236266682545344
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,31,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,31,0.02306666721900304
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,63,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,63,0.023039999107519787
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,255,0.018090666582187016
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,127,0.028330666323502857
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,255,0.04966400067011515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,511,0.02918400118748347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,511,0.09352533022562663
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,1023,0.052095999320348106
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,1023,0.20992000897725424
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,2047,0.10052266716957092
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,2047,0.35703468322753906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,4095,0.19899733861287436
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,4095,0.706559975941976
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,8191,0.38809601465861004
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,8191,1.426479975382487
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,16383,0.8381386597951254
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,1,0.010202666744589806
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,16383,4.237653414408366
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,1,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,3,0.010421333213647207
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,3,0.020143999407688778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,7,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,7,0.021333334346612293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,15,0.023210667073726654
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,15,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,31,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,31,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,63,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,63,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,127,0.029834667841593426
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,255,0.01811733345190684
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,255,0.051029334465662636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,511,0.029509333272775013
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,511,0.09847467144330342
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,1023,0.051029334465662636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,1023,0.18210132916768393
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,2047,0.10188266634941101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,2047,0.35703468322753906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,4095,0.2001919945081075
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,4095,0.7070773442586263
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,8191,0.39372801780700684
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,8191,1.4271252950032551
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,1,0.0100853331387043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,16383,0.8692053159077963
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,1,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,3,0.010357333347201347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,3,0.021333334346612293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,16383,4.241717338562012
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,7,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,7,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,15,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,15,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,31,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,31,0.023541333774725597
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,63,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,63,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,127,0.030378667016824085
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,255,0.01791999985774358
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,511,0.09386666615804036
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,255,0.05341866612434387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,511,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,1023,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,2047,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,1023,0.18212266763051352
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,2047,0.3570293188095093
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,4095,0.02611200014750163
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,4095,0.706559975941976
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,8191,0.03822933385769526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,16383,0.05495466788609823
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,8191,1.4277973175048828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,1,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,1,0.020992000897725422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,16383,4.246869405110677
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,3,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,3,0.021333334346612293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,7,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,7,0.021701333423455555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,15,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,15,0.023599999646345775
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,31,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,31,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,63,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,63,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,127,0.02958400050799052
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,255,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,255,0.05101333558559418
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,511,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,511,0.09453333417574565
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,1023,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,1023,0.18210132916768393
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,2047,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,2047,0.35703468322753906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,4095,0.03379199902216593
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,4095,0.7069013118743896
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,8191,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,16383,0.22715733448664346
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,8191,1.4272853533426921
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,1,0.1104213297367096
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,16383,4.242090543111165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,1,0.3336533308029175
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,3,0.11588266491889954
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,3,0.35225598017374676
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,7,0.12134400010108948
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,7,0.36932265758514404
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,15,0.15308800339698792
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,15,0.4978346824645996
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,31,0.15308800339698792
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,63,0.15667200088500977
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,31,0.49766401449839276
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,127,0.1960960030555725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,63,0.4981333414713542
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,255,0.33058132727940875
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,127,0.6444373528162638
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,511,0.6251519918441772
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,255,1.2020053068796794
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,1023,1.2253866990407307
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,511,2.3284053802490234
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,1,0.11076266566912334
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,1,0.33536001046498615
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,3,0.11549866199493408
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,1023,4.597418785095215
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,3,0.3534506559371948
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,7,0.12150933345158894
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,15,0.1534293293952942
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,7,0.3705173333485921
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,15,0.4978346824645996
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,31,0.15359999736150107
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,31,0.4978346824645996
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,63,0.15769599874814352
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,63,0.4984000126520793
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,127,0.20652800798416138
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,255,0.34355199337005615
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,127,0.6447786490122477
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,511,0.6447733243306478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,255,1.202346642812093
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,511,2.327552000681559
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,1023,1.2794880072275798
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,1,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,1023,4.596906661987305
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,3,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,3,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,1,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,7,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,7,0.02918400118748347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,15,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,15,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,31,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,31,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,63,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,127,0.03379199902216593
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,63,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,255,0.04761599997679392
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,127,0.02867199977238973
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,255,0.03925333420435587
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,511,0.06809600194295247
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,511,0.059392000238100685
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,1023,0.24678399165471396
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,1023,0.09267200032869975
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,1,0.032085334261258446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,1,0.040106666584809623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,3,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,3,0.032085334261258446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,7,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,7,0.040106666584809623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,15,0.040106666584809623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,15,0.032085334261258446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,31,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,31,0.032085334261258446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,63,0.032085334261258446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,127,0.040106666584809623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,63,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,127,0.04437333345413208
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,255,0.05120000243186951
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,255,0.056661332647005715
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,511,0.23176532983779907
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,511,0.09489066402117412
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,1023,0.4787199894587199
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,1023,0.23910399278004965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,fp8,1,0.662389318148295
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,1,0.21640533208847046
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,3,0.22698666652043661
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,fp8,3,0.6993920008341471
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,7,0.23859200874964395
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,fp8,7,0.7323306401570638
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,15,0.3025919993718465
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,31,0.3027626673380534
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,fp8,15,1.005226691563924
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,63,0.3089066743850708
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,fp8,31,0.9885120391845703
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,fp8,63,0.9898773034413656
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,127,0.38656000296274823
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,fp8,127,1.281215985616048
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,255,0.651093324025472
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,1,0.21640533208847046
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,fp8,255,2.3881386121114097
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,fp8,1,0.665610671043396
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,fp8,3,0.7038293679555258
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,3,0.22660799821217856
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,7,0.23961599667867026
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,fp8,7,0.7365972995758057
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,15,0.30293333530426025
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,fp8,15,0.9890133539835612
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,31,0.30344533920288086
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,fp8,31,0.9886720180511475
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,63,0.3107680082321167
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,127,0.4097706476847331
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,fp8,63,0.9902079900105795
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,255,0.6736213366190592
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,fp8,127,1.2811946868896484
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,1,0.04369066655635834
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,fp8,255,2.3883093198140464
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,fp8,1,0.053247998158137
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,3,0.04369066655635834
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,fp8,3,0.05273599922657013
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,fp8,7,0.053077335158983864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,15,0.04369066655635834
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,7,0.04351999859015147
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,fp8,15,0.053247998158137
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,31,0.04369066655635834
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,fp8,31,0.053077335158983864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,63,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,fp8,63,0.05358933409055074
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,127,0.05222400029500326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,fp8,127,0.06075733403364817
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,255,0.06365866462389629
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,fp8,255,0.07031466563542683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,1,0.05751466751098633
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,fp8,1,0.07287466526031494
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,3,0.05717333157857259
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,fp8,3,0.07202133536338806
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,7,0.05717333157857259
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,fp8,7,0.0721919983625412
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,fp8,15,0.07202133536338806
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,31,0.05717333157857259
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,fp8,31,0.07253333429495494
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,15,0.05751466751098633
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,fp8,63,0.07202133536338806
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,63,0.05819733440876007
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,127,0.07355733215808868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,fp8,127,0.08106666803359985
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,255,0.24320000410079956
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,fp8,255,0.10308266679445903
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,float16,1,0.42769066492716473
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,fp8,1,1.3204480012257893
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,float16,3,0.44970667362213135
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,fp8,3,1.3972479502360027
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,float16,7,0.47288533051808673
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,fp8,7,1.460026741027832
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,float16,15,0.6016000111897787
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,fp8,15,1.9694933891296387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,float16,31,0.6016000111897787
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,fp8,31,1.9690346717834473
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,float16,63,0.6142239967981974
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,float16,127,0.7623519897460938
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,fp8,63,1.9722240765889485
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,float16,1,0.42803200085957843
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,fp8,127,2.554368019104004
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,float16,3,0.4486773411432902
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,fp8,1,1.327285369237264
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,float16,7,0.47547733783721924
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,fp8,3,1.4057812690734863
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,fp8,7,1.4706613222757976
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,float16,15,0.6028053363164266
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,fp8,15,1.970688025156657
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,float16,31,0.6034773190816244
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,float16,63,0.6164373159408569
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,fp8,31,1.969663937886556
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,fp8,63,1.9727360407511394
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,float16,127,0.8085866769154867
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,float16,1,0.07884799937407176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,fp8,127,2.553856054941813
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,fp8,1,0.0962559978167216
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,fp8,3,0.0962559978167216
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,float16,3,0.07867733140786488
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,float16,7,0.07850666840871175
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,fp8,7,0.09693866968154907
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,fp8,15,0.09676800171534221
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,float16,15,0.07850666840871175
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,fp8,31,0.09659733374913533
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,float16,31,0.07884799937407176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,float16,63,0.07867733140786488
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,fp8,63,0.09676800171534221
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,float16,127,0.16947199900945029
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,fp8,127,0.1109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,float16,1,0.10683733224868774
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,float16,3,0.10683733224868774
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,float16,7,0.10666666428248088
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,fp8,3,0.13550933202107748
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,float16,15,0.106495996316274
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,fp8,1,0.13550933202107748
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,fp8,7,0.13550933202107748
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,fp8,15,0.13550933202107748
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,fp8,31,0.13550933202107748
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,float16,31,0.10922666390736897
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,float16,63,0.16913066307703653
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,1,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,fp8,63,0.13550933202107748
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,float16,127,0.274944007396698
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,fp8,127,0.1925119956334432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,1,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,3,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,7,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,3,0.02935466667016347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,7,0.030207999050617218
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,15,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,15,0.03839999934037527
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,31,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,31,0.038912000755469
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,63,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,63,0.03839999934037527
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,127,0.01658133293191592
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,127,0.04915200173854828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,255,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,255,0.08942932883898418
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,511,0.046575998266537987
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,511,0.17116800944010416
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,1023,0.08586133519808452
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,1023,0.33433600266774494
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,2047,0.16383999586105347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,4095,0.36113067468007404
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,2047,0.6601386864980062
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,4095,1.3116052945454915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,8191,0.7142399946848551
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,1,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,8191,2.8235092163085938
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,16383,2.0906666119893393
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,1,0.029178666571776073
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,3,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,3,0.029909332593282063
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,16383,8.451242446899414
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,7,0.010570666442314783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,7,0.030741333961486816
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,15,0.013295999417702356
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,15,0.03857066730658213
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,31,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,63,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,31,0.03857066730658213
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,63,0.03855466594298681
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,127,0.017407999684413273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,127,0.04966400067011515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,255,0.02867199977238973
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,255,0.08942932883898418
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,511,0.04778666794300079
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,511,0.17117865880330405
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,1023,0.08686932921409607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,1023,0.33433600266774494
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,2047,0.16759467124938965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,2047,0.6596266825993856
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,4095,0.3978186845779419
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,4095,1.3114240169525146
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,8191,0.8024746576944987
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,8191,2.822655995686849
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,16383,2.164741357167562
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,1,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,1,0.029525332152843475
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,3,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,3,0.030378667016824085
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,7,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,7,0.03089066594839096
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,16383,8.457733154296875
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,15,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,15,0.03957866628964742
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,31,0.03942399968703588
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,31,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,63,0.013642666240533194
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,63,0.03908266623814901
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,127,0.017237332959969837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,127,0.04915200173854828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,255,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,511,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,1023,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,255,0.08959999680519104
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,511,0.17100799083709717
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,2047,0.026965332527955372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,1023,0.33433600266774494
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,2047,0.6594880024592081
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,4095,0.040618665516376495
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,8191,0.059392000238100685
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,4095,1.31222931543986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,16383,0.2402986685434977
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,1,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,1,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,3,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,8191,2.8242079416910806
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,16383,8.466431935628256
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,3,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,7,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,7,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,31,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,31,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,63,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,63,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,255,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,255,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,511,0.015872000406185787
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,511,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,1023,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,1023,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,2047,0.03498666733503342
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,2047,0.04215466479460398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,4095,0.07406933108965556
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,4095,0.05717333157857259
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,8191,0.2213546633720398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,8191,0.13960533340771994
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,16383,0.43298133214314777
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,16383,0.2757973273595174
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,float16,1,0.8507733345031738
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,float16,3,0.8946346441904703
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,fp8,1,2.639530658721924
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,fp8,3,2.7970558802286782
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,float16,7,0.942250649134318
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,fp8,7,2.914645195007324
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,float16,15,1.1996160348256428
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,fp8,15,3.932330767313639
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,float16,31,1.1999573707580566
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,float16,63,1.2250453631083171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,fp8,31,3.930624008178711
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,float16,1,0.8509439627329508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,fp8,63,3.9362026850382485
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,float16,3,0.8927573362986246
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,fp8,1,2.651818593343099
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,fp8,3,2.817194620768229
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,float16,7,0.9471999804178873
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,float16,15,1.201210657755534
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,fp8,7,2.94161065419515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,fp8,15,3.934720039367676
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,float16,31,1.2034293015797932
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,float16,63,1.2378453413645427
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,fp8,31,3.932330767313639
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,float16,1,0.18705066045125326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,fp8,63,3.943098704020182
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,fp8,1,0.21555199225743613
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,float16,3,0.20411733786265054
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,fp8,3,0.21811199188232422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,float16,7,0.211626668771108
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,float16,15,0.22698666652043661
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,fp8,7,0.22272000710169473
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,fp8,15,0.2285226583480835
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,float16,31,0.2387626568476359
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,fp8,31,0.24115200837453207
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,float16,63,0.279039998849233
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,float16,1,0.25036799907684326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,fp8,63,0.2629973292350769
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,fp8,1,0.3065173427263896
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,float16,3,0.26316799720128375
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,fp8,3,0.32614399989446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,float16,7,0.28706133365631104
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,fp8,7,0.3179519971211751
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,float16,15,0.31061333417892456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,fp8,15,0.3259733319282532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,float16,31,0.3222186764081319
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,fp8,31,0.36420265833536786
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,float16,63,0.3723946809768677
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,fp8,63,0.353279987970988
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,float16,1,1.7986666361490886
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,float16,3,1.8737492561340332
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,fp8,1,5.357056299845378
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,float16,7,1.948362668355306
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,fp8,3,5.6861012776692705
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,fp8,7,5.906944274902344
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,float16,15,2.434901396433512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,fp8,15,7.886538823445638
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,float16,31,2.4499200185139975
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,float16,1,1.8510506947835286
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,float16,3,1.9012373288472493
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,fp8,1,5.383839925130208
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,fp8,31,7.884629567464192
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,float16,7,1.9817652702331543
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,fp8,3,5.719519933064778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,float16,15,2.442410628000895
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,fp8,7,5.959679921468099
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,float16,1,0.5138773520787557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,fp8,15,7.889279683430989
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,float16,31,2.453674634297689
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,fp8,1,0.5838506619135538
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,fp8,31,7.888042449951172
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,float16,3,0.5145599842071533
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,fp8,3,0.5814613501230875
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,float16,7,0.5118293364842733
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,fp8,7,0.5809493462244669
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,float16,15,0.5060266653696696
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,fp8,15,0.579584002494812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,float16,31,0.5092693169911703
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,fp8,31,0.5760000149408976
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,float16,1,0.6674773693084717
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,fp8,1,0.7613440354665121
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,float16,3,0.6758399804433187
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,fp8,3,0.7621973355611166
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,float16,7,0.6719146569569906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,fp8,7,0.762880007425944
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,fp8,15,0.7589546839396158
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,1,0.017237332959969837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,float16,15,0.6493866840998331
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,float16,31,0.6517759958902994
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,fp8,31,0.7483733495076498
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,1,0.04607999821503957
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,3,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,3,0.0481279989083608
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,7,0.01826133330663045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,7,0.050698667764663696
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,15,0.06846400101979573
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,15,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,31,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,63,0.023039999107519787
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,31,0.06877866884072621
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,63,0.06911999980608623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,127,0.02918400118748347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,127,0.08787733316421509
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,255,0.048469334840774536
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,255,0.16196266810099283
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,511,0.0846506655216217
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,511,0.3107840021451314
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,1023,0.16179200013478598
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,1023,0.6113280057907104
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,2047,0.318122665087382
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,2047,1.213098684946696
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,4095,0.7306239604949951
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,1,0.017237332959969837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,8191,1.4900906880696614
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,4095,2.4144213994344077
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,1,0.04693333307902018
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,8191,5.273258527119954
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,3,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,3,0.048138668139775596
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,7,0.018090666582187016
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,7,0.05120000243186951
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,15,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,15,0.06860800087451935
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,31,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,31,0.0687306672334671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,63,0.023381332556406658
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,63,0.06929066777229309
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,127,0.030250666042168934
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,255,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,127,0.08891733487447102
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,255,0.16201066970825195
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,511,0.08823466300964355
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,511,0.31112533807754517
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,1023,0.16776533921559653
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,1023,0.6113280057907104
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,2047,0.3253386616706848
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,2047,1.2139519850413005
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,4095,0.7630506356557211
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,8191,1.594879945119222
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,4095,2.4144213994344077
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,1,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,1,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,8191,5.273599942525228
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,3,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,7,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,7,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,15,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,15,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,31,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,63,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,63,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,127,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,127,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,255,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,255,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,511,0.01791999985774358
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,511,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,1023,0.027136000494162243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,1023,0.025941332181294758
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,2047,0.04215466479460398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,2047,0.04334933559099833
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,4095,0.06724266707897186
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,4095,0.07628799974918365
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,8191,0.23278933763504028
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,8191,0.14165332913398743
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,1,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,1,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,3,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,7,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,15,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,15,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,31,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,63,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,63,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,127,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,127,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,255,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,255,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,511,0.023039999107519787
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,511,0.01621333385507266
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,1023,0.03703466554482778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,1023,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,2047,0.04232533276081085
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,4095,0.22254933913548788
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,2047,0.05580799778302511
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,4095,0.07509333391984303
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,8191,0.43144532044728595
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,8191,0.21691733598709106
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,1,0.023552000522613525
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,1,0.06830400228500366
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,3,0.06809600194295247
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,3,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,7,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,7,0.07133866846561432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,15,0.02495466669400533
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,15,0.0747519979874293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,31,0.031061333914597828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,31,0.09762133161226909
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,63,0.0314026673634847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,63,0.09693866968154907
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,127,0.03242666771014532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,127,0.09301333626111348
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,255,0.04164266586303711
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,255,0.1276586651802063
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,511,0.06911999980608623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,511,0.23244800170262656
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,1023,0.12591999769210815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,1023,0.4480106830596924
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,2047,0.2373973329861959
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,2047,0.8813227017720541
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,4095,0.5216960112253824
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,4095,1.7568426132202148
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,float16,1,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,fp8,1,0.06860800087451935
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,float16,3,0.02350933353106181
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,fp8,3,0.06877866884072621
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,fp8,7,0.0694400022427241
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,float16,15,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,float16,7,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,fp8,15,0.0988159974416097
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,float16,31,0.031061333914597828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,fp8,31,0.0981333355108897
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,float16,63,0.03126399964094162
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,fp8,63,0.0981333355108897
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,float16,127,0.03242666771014532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,fp8,127,0.09779199957847595
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,float16,255,0.043007999658584595
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,fp8,255,0.1262933313846588
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,float16,511,0.0716480016708374
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,fp8,511,0.23278933763504028
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,float16,1023,0.12970667084058127
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,fp8,1023,0.4483413298924764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,float16,2047,0.24166399240493774
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,fp8,2047,0.8809813658396403
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,float16,4095,0.5435733397801717
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,64,0,1,float16,fp8,4095,1.7571840286254883
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,1,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,3,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,3,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,7,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,15,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,15,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,63,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,31,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,63,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,255,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,1023,0.016042667130629223
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,2047,0.04863999783992767
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,2047,0.025429333249727886
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,4095,0.06656000018119812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,4095,0.04215466479460398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,511,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,255,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,511,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,1023,0.034474665919939675
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,1,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,3,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,1,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,15,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,7,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,31,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,3,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,63,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,7,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,15,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,127,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,255,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,1023,0.03925333420435587
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,63,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,31,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,1023,0.025941332181294758
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,2047,0.0576853354771932
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,127,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,255,0.018944000204404194
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,2047,0.04351999859015147
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,4095,0.20514132579167685
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,511,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,511,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,4095,0.07628799974918365
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,1,0.007360000163316727
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,1,0.010768000036478043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,3,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,15,0.007327999919652939
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,7,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,15,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,31,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,63,0.013514666507641474
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,31,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,63,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,127,0.008687999720374743
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,255,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,255,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,511,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,511,0.030378667016824085
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,1023,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,1023,0.04607999821503957
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,2047,0.025258667767047882
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,2047,0.06417066852251689
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,4095,0.03839999934037527
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,4095,0.11434666315714519
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,8191,0.06229333579540253
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,8191,0.19473065932591757
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,16383,0.10700800021489461
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,16383,0.3696639935175578
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,fp8,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,float16,3,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,fp8,3,0.010778666784365972
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,float16,7,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,fp8,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,float16,15,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,fp8,15,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,float16,31,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,fp8,31,0.011946666985750198
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,float16,63,0.00821333316465219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,fp8,63,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,fp8,127,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,float16,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,fp8,511,0.03054933249950409
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,float16,511,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,fp8,255,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,float16,1023,0.020138667275508244
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,fp8,1023,0.04351999859015147
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,float16,2047,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,fp8,2047,0.06519466638565063
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,float16,4095,0.03857066730658213
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,fp8,4095,0.11895466844240825
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,float16,8191,0.06195199986298879
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,fp8,8191,0.19490132729212442
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,float16,16383,0.10854400197664897
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,64,0,1,float16,fp8,16383,0.3690986633300781
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,1,0.007674666742483775
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,1,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,3,0.007141333073377609
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,3,0.011109333485364914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,15,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,31,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,31,0.011546666423479715
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,63,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,63,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,127,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,255,0.008559999987483025
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,255,0.013631999492645264
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,511,0.030720000465710957
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,511,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,7,0.011621333658695221
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,1023,0.020469332734743755
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,1023,0.044400001565615334
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,2047,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,2047,0.06489600241184235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,4095,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,4095,0.10806399583816528
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,8191,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,8191,0.1950719952583313
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,16383,0.01621333385507266
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,1,0.007482666522264481
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,16383,0.3689813216527303
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,3,0.0074879998962084455
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,3,0.0315733328461647
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,7,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,7,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,31,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,31,0.01157333329319954
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,63,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,63,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,255,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,255,0.00867733359336853
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,511,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,511,0.03293866664171219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,1023,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,1023,0.04642133414745331
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,15,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,2047,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,2047,0.06486933430035909
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,4095,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,4095,0.10820266604423523
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,8191,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,8191,0.19541333119074503
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,16383,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,16383,0.370192011197408
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,1,0.006709333509206772
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,1,0.017914666483799618
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,3,0.006826666494210561
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,3,0.018090666582187016
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,7,0.006826666494210561
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,7,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,15,0.007018666714429855
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,15,0.018090666582187016
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,31,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,31,0.0194560003777345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,63,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,63,0.023210667073726654
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,127,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,127,0.023381332556406658
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,255,0.008752000207702318
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,255,0.024586667617162068
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,511,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,511,0.031231999397277832
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,1023,0.01565333331624667
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,1023,0.056320001681645714
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,2047,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,2047,0.1063253382841746
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,4095,0.04369066655635834
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,4095,0.20497065782546997
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,8191,0.08328533172607422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,8191,0.3510613441467285
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,16383,0.2032639980316162
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,float16,1,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,16383,0.6946132977803549
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,fp8,1,0.019285333653291065
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,float16,3,0.006655999769767125
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,fp8,3,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,float16,7,0.00697066696981589
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,fp8,7,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,float16,15,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,fp8,15,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,float16,31,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,fp8,31,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,float16,63,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,fp8,63,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,float16,127,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,float16,255,0.008394666636983553
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,fp8,127,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,fp8,255,0.02628266563018163
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,float16,511,0.009925333162148794
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,fp8,511,0.030378667016824085
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,float16,1023,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,fp8,1023,0.05614933371543884
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,float16,2047,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,fp8,2047,0.11337066690127055
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,float16,4095,0.04505600035190582
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,fp8,4095,0.1790293256441752
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,float16,8191,0.08482133348782857
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,fp8,8191,0.35123201211293537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,float16,16383,0.2037386695543925
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,64,0,1,float16,fp8,16383,0.695466677347819
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,1,0.01932266727089882
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,3,0.006655999769767125
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,7,0.006826666494210561
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,7,0.0194560003777345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,15,0.006826666494210561
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,15,0.019285333653291065
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,31,0.007525333513816197
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,31,0.02025066688656807
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,63,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,63,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,127,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,1,0.006826666494210561
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,127,0.023904000719388325
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,255,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,255,0.025429333249727886
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,3,0.018944000204404194
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,511,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,511,0.03278400003910065
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,1023,0.05120000243186951
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,1023,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,2047,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,2047,0.09457066655158997
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,4095,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,4095,0.17937066157658896
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,8191,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,8191,0.35123201211293537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,1,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,16383,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,16383,0.6949386596679688
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,1,0.019285333653291065
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,3,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,3,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,7,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,15,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,7,0.019285333653291065
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,15,0.019285333653291065
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,31,0.0074879998962084455
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,31,0.020992000897725422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,63,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,63,0.025429333249727886
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,127,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,127,0.024746666351954143
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,255,0.008682666967312494
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,255,0.025600001215934753
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,511,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,511,0.034874667723973594
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,1023,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,1023,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,2047,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,2047,0.09437867005666097
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,4095,0.17937066157658896
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,4095,0.015872000406185787
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,8191,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,8191,0.3519146839777629
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,16383,0.03498666733503342
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,1,0.043178667624791466
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,16383,0.695466677347819
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,1,0.12782933314641318
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,3,0.043007999658584595
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,3,0.12919466694196066
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,7,0.04488533238569895
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,7,0.13431466619173685
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,15,0.04659200211366018
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,15,0.14250666896502176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,31,0.19131733973821005
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,31,0.05870933334032694
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,63,0.05905066430568695
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,63,0.18414932489395142
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,127,0.061247999469439186
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,127,0.18874132633209229
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,255,0.07850666840871175
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,511,0.13312000036239624
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,255,0.24541866779327393
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,511,0.4575573205947876
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,1023,0.24883200724919638
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,1023,0.8840693632761637
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,2047,0.47189335028330487
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,2047,1.7435305913289387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,2,64,0,1,float16,float16,1,0.043178667624791466
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,2,64,0,1,float16,fp8,1,0.12820266683896384
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,2,64,0,1,float16,float16,3,0.04334933559099833
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,2,64,0,1,float16,float16,7,0.04472533365090688
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,2,64,0,1,float16,fp8,3,0.12800000111262003
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,2,64,0,1,float16,fp8,7,0.13636266191800436
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,2,64,0,1,float16,float16,15,0.0467199981212616
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,2,64,0,1,float16,fp8,15,0.1430186629295349
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,2,64,0,1,float16,float16,31,0.05870933334032694
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,2,64,0,1,float16,fp8,31,0.19131733973821005
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,2,64,0,1,float16,float16,63,0.059392000238100685
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,2,64,0,1,float16,fp8,63,0.1889280080795288
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,2,64,0,1,float16,float16,127,0.06109866499900818
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,2,64,0,1,float16,fp8,127,0.18961066007614136
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,2,64,0,1,float16,float16,255,0.08041066428025563
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,2,64,0,1,float16,fp8,255,0.24524799982706705
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,2,64,0,1,float16,float16,511,0.13772799571355185
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,2,64,0,1,float16,fp8,511,0.45823999245961505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,2,64,0,1,float16,float16,1023,0.25224532683690387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,2,64,0,1,float16,fp8,1023,0.8836959997812907
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,2,64,0,1,float16,float16,2047,0.4848639965057373
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,1,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,2,64,0,1,float16,fp8,2047,1.7433600425720215
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,fp8,1,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,fp8,3,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,3,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,7,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,fp8,7,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,15,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,fp8,15,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,31,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,fp8,31,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,63,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,fp8,63,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,127,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,fp8,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,255,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,fp8,255,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,511,0.03498666733503342
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,fp8,511,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,1023,0.0481279989083608
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,fp8,1023,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,2047,0.06724266707897186
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,fp8,2047,0.04625066618124644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,1,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,1,0.017237332959969837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,3,0.016895999511082966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,3,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,7,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,7,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,15,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,15,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,31,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,63,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,31,0.016895999511082966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,63,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,127,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,127,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,255,0.031061333914597828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,255,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,511,0.04113066693147024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,511,0.03259733319282532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,1023,0.0576853354771932
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,1023,0.04744533201058706
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,2047,0.20804266134897867
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,1,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,2047,0.08089600006739299
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,1,0.026474667092164356
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,3,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,3,0.059392000238100685
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,7,0.009919999788204828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,7,0.026965332527955372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,15,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,15,0.027136000494162243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,31,0.028501334289709728
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,31,0.00985599992175897
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,63,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,63,0.03498666733503342
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,127,0.011946666985750198
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,127,0.03549866626660029
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,255,0.012618667135636011
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,255,0.037717332442601524
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,511,0.014874666929244995
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,511,0.04761599997679392
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,1023,0.027642667293548584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,1023,0.07850666840871175
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,2047,0.04647466540336609
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,2047,0.16179200013478598
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,4095,0.08601599931716919
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,4095,0.2676053245862325
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,8191,0.15479466319084167
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,8191,0.5251413186391195
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,16383,0.32238932450612384
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,float16,1,0.009877333417534828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,16383,1.0408960183461506
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,fp8,1,0.027477333943049114
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,float16,3,0.0099093330403169
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,float16,7,0.009914666414260864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,fp8,3,0.027647999425729115
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,fp8,7,0.027615999182065327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,float16,15,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,fp8,15,0.027477333943049114
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,float16,31,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,fp8,31,0.029711998999118805
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,float16,63,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,fp8,63,0.03310933212439219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,float16,127,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,fp8,127,0.03618133316437403
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,float16,255,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,float16,511,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,fp8,255,0.035029334326585136
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,fp8,511,0.049322664737701416
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,float16,1023,0.026965332527955372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,fp8,1023,0.07521600027879079
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,float16,2047,0.04778666794300079
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,fp8,2047,0.13980799913406372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,float16,4095,0.08669867118199666
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,fp8,4095,0.2681173284848531
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,float16,8191,0.16590933005015054
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,fp8,8191,0.5251413186391195
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,float16,16383,0.3351893424987793
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,1,0.009888000165422758
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,64,0,1,float16,fp8,16383,1.040885368982951
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,1,0.02696000039577484
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,3,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,3,0.027647999425729115
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,7,0.010053333515922228
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,7,0.027818667391935985
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,15,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,15,0.027818667391935985
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,31,0.009701333319147428
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,31,0.02867199977238973
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,63,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,63,0.03430933256944021
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,127,0.011557333171367645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,127,0.034448000291983284
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,255,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,255,0.037717332442601524
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,511,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,511,0.04369066655635834
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,1023,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,1023,0.07594666878382365
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,2047,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,2047,0.14045866330464682
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,4095,0.018944000204404194
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,4095,0.2681173284848531
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,8191,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,8191,0.5250826676686605
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,16383,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,1,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,16383,1.0416266918182373
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,1,0.027647999425729115
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,3,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,3,0.027477333943049114
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,7,0.0099093330403169
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,15,0.010064000263810158
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,7,0.027658666173617046
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,15,0.027306665976842243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,31,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,31,0.030394665896892548
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,63,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,63,0.03549866626660029
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,127,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,127,0.03583999971548716
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,511,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,255,0.03549866626660029
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,511,0.04437333345413208
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,1023,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,2047,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,1023,0.07594666878382365
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,2047,0.1397760013739268
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,4095,0.025941332181294758
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,4095,0.2686079939206441
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,8191,0.03737599899371465
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,8191,0.5256533225377401
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,16383,0.059562668204307556
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,1,0.08243733147780101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,16383,1.0419200261433919
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,1,0.2481493353843689
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,3,0.08260266482830048
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,3,0.24797866741816202
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,7,0.08563199639320374
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,15,0.0897706647713979
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,7,0.26743467648824054
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,31,0.11315199732780457
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,15,0.2757973273595174
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,31,0.3691519896189372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,63,0.11434666315714519
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,63,0.3689813216527303
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,127,0.11878400047620137
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,127,0.371018648147583
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,255,0.15172266960144043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,255,0.4804266691207886
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,511,0.2584213415781657
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,511,0.8987306753794352
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,1023,0.47973867257436115
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,2,64,0,1,float16,float16,1,0.08260266482830048
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,2,64,0,1,float16,fp8,1,0.24985599517822266
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,1023,1.737386703491211
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,2,64,0,1,float16,float16,3,0.08260266482830048
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,2,64,0,1,float16,fp8,3,0.24883200724919638
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,2,64,0,1,float16,float16,7,0.08567466338475545
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,2,64,0,1,float16,fp8,7,0.26368000109990436
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,2,64,0,1,float16,float16,15,0.08942932883898418
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,2,64,0,1,float16,fp8,15,0.27688000599543255
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,2,64,0,1,float16,float16,31,0.11349333326021831
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,2,64,0,1,float16,fp8,31,0.3694933255513509
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,2,64,0,1,float16,float16,63,0.11434666315714519
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,2,64,0,1,float16,fp8,63,0.3696639935175578
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,2,64,0,1,float16,float16,127,0.11844266454378764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,2,64,0,1,float16,float16,255,0.15615999698638916
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,2,64,0,1,float16,fp8,127,0.37085866928100586
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,2,64,0,1,float16,float16,511,0.26657599210739136
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,2,64,0,1,float16,fp8,255,0.48076800505320233
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,2,64,0,1,float16,fp8,511,0.8985599676767985
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,2,64,0,1,float16,float16,1023,0.4843519926071167
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,1,0.021503999829292297
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,2,64,0,1,float16,fp8,1023,1.736421267191569
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,fp8,1,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,fp8,3,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,3,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,7,0.020992000897725422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,fp8,7,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,15,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,fp8,15,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,31,0.021333334346612293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,63,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,fp8,31,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,fp8,63,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,127,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,fp8,127,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,255,0.037205333511034645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,fp8,255,0.025941332181294758
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,511,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,fp8,511,0.03618133316437403
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,1,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,1023,0.0682666649421056
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,1,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,fp8,1023,0.052906667192777
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,3,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,3,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,7,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,7,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,15,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,31,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,15,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,31,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,63,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,63,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,127,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,127,0.03669333209594091
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,255,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,255,0.03822933385769526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,511,0.053930665055910744
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,511,0.05853866537412008
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,1023,0.20787199338277182
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,1023,0.0885759989420573
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,float16,1,0.16059733430544534
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,fp8,1,0.4944213231404622
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,float16,3,0.16076800227165222
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,fp8,3,0.4935679833094279
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,float16,7,0.16708266735076904
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,fp8,7,0.5234346787134806
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,float16,15,0.1754186749458313
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,fp8,15,0.5464959939320883
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,float16,31,0.22272000710169473
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,fp8,31,0.7328586578369141
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,float16,63,0.22476800282796225
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,fp8,63,0.7326239744822184
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,float16,127,0.23381332556406656
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,fp8,127,0.73471999168396
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,float16,255,0.29815467198689777
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,2,64,0,1,float16,float16,1,0.16145066420237222
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,fp8,255,0.9540266990661621
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,2,64,0,1,float16,fp8,1,0.497322678565979
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,2,64,0,1,float16,float16,3,0.16145066420237222
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,2,64,0,1,float16,fp8,3,0.49646933873494464
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,2,64,0,1,float16,float16,7,0.16674133141835532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,2,64,0,1,float16,fp8,7,0.5258239905039469
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,2,64,0,1,float16,float16,15,0.17527467012405396
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,2,64,0,1,float16,fp8,15,0.5495466788609823
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,2,64,0,1,float16,float16,31,0.2228906750679016
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,2,64,0,1,float16,fp8,31,0.7332106431325277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,2,64,0,1,float16,float16,63,0.22476800282796225
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,2,64,0,1,float16,fp8,63,0.7328426837921143
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,2,64,0,1,float16,float16,127,0.23278933763504028
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,2,64,0,1,float16,fp8,127,0.7348693211873373
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,2,64,0,1,float16,float16,255,0.30670400460561115
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,float16,1,0.03942399968703588
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,2,64,0,1,float16,fp8,255,0.9536853631337484
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,fp8,1,0.04113066693147024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,float16,3,0.040106666584809623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,fp8,3,0.04095999896526337
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,float16,7,0.040106666584809623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,fp8,7,0.04147200038035711
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,float16,15,0.040106666584809623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,fp8,15,0.040618665516376495
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,float16,31,0.03925333420435587
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,fp8,31,0.04113066693147024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,float16,63,0.040106666584809623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,fp8,63,0.040789333482583366
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,float16,127,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,fp8,127,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,float16,255,0.04727466901143392
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,float16,1,0.05341866612434387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,fp8,255,0.04898133377234141
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,fp8,1,0.054101333022117615
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,float16,3,0.05341866612434387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,fp8,3,0.054272000988324486
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,float16,7,0.05341866612434387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,fp8,7,0.05444266895453135
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,float16,15,0.05341866612434387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,fp8,15,0.054272000988324486
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,float16,31,0.05341866612434387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,fp8,31,0.053930665055910744
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,float16,63,0.053077335158983864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,fp8,63,0.054272000988324486
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,float16,127,0.05358933409055074
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,fp8,127,0.0679253339767456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,float16,255,0.06894933183987935
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,fp8,255,0.08157866696516673
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,float16,1,0.3176106611887614
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,float16,3,0.31692800919214886
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,fp8,1,0.9857707023620605
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,fp8,3,0.9849173227945963
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,float16,7,0.3293866713841756
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,fp8,7,1.0470399856567383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,float16,15,0.3657386700312297
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,fp8,15,1.0897066593170166
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,float16,31,0.44185598691304523
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,float16,63,0.44629331429799396
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,fp8,31,1.4568106333414714
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,fp8,63,1.4580052693684895
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,float16,127,0.463701327641805
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,2,64,0,1,float16,float16,1,0.31948800881703693
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,2,64,0,1,float16,fp8,1,0.9900373617808024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,2,64,0,1,float16,float16,3,0.31914667288462323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,fp8,127,1.4615893363952637
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,2,64,0,1,float16,fp8,3,0.9920852979024252
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,2,64,0,1,float16,float16,7,0.3290453354517619
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,2,64,0,1,float16,fp8,7,1.051695982615153
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,2,64,0,1,float16,float16,15,0.34645334879557294
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,2,64,0,1,float16,float16,31,0.44151465098063153
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,2,64,0,1,float16,fp8,15,1.098581314086914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,2,64,0,1,float16,float16,63,0.4456106821695964
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,2,64,0,1,float16,fp8,31,1.4578347206115723
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,2,64,0,1,float16,float16,127,0.4612746636072795
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,2,64,0,1,float16,fp8,63,1.4609066645304363
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,float16,1,0.0721919983625412
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,2,64,0,1,float16,fp8,127,1.462272008260091
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,fp8,1,0.07321600119272868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,float16,3,0.0718506673971812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,fp8,3,0.07338666419188182
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,float16,7,0.07236266632874806
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,fp8,7,0.07321600119272868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,float16,15,0.07202133536338806
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,fp8,15,0.07355733215808868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,fp8,31,0.07321600119272868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,float16,31,0.07202133536338806
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,float16,63,0.0718506673971812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,fp8,63,0.07338666419188182
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,float16,127,0.07253333429495494
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,fp8,127,0.07441066702206929
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,float16,1,0.09966933727264404
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,fp8,1,0.10120532910029094
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,float16,3,0.09949866930643718
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,fp8,3,0.10120532910029094
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,float16,7,0.09949866930643718
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,float16,15,0.09983999530474345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,fp8,7,0.10103467106819153
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,float16,31,0.09966933727264404
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,fp8,15,0.10154666503270467
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,float16,63,0.09949866930643718
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,fp8,31,0.10086400310198466
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,fp8,63,0.10086400310198466
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,1,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,fp8,127,0.1276586651802063
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,float16,127,0.09983999530474345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,1,0.019626667102177937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,3,0.008992000172535578
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,3,0.0194560003777345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,7,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,7,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,15,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,15,0.021333334346612293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,31,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,31,0.02985599885384242
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,63,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,63,0.029696000119050343
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,127,0.011381333072980246
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,127,0.029877332349618275
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,255,0.0138026662170887
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,255,0.037717332442601524
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,511,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,511,0.06860800087451935
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,1023,0.034474665919939675
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,1023,0.1302186648050944
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,2047,0.06401599943637848
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,2047,0.2539680004119873
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,4095,0.12987732887268066
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,4095,0.5015626748402914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,8191,0.2578773299853007
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,8191,1.0951680342356365
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,16383,0.8033280372619629
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,float16,1,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,fp8,1,0.019808000574509304
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,float16,3,0.00902399979531765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,16383,3.1933441162109375
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,fp8,3,0.019797333826621372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,float16,7,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,fp8,7,0.020992000897725422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,float16,15,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,float16,31,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,fp8,15,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,fp8,31,0.029696000119050343
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,float16,63,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,fp8,63,0.03054933249950409
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,float16,127,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,fp8,127,0.030720000465710957
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,float16,255,0.013477332890033722
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,fp8,255,0.03841600070397059
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,float16,511,0.021162666380405426
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,fp8,511,0.06877866884072621
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,float16,1023,0.03532800078392029
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,fp8,1023,0.13056000073750815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,float16,2047,0.06400000055631001
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,fp8,2047,0.25412267446517944
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,float16,4095,0.13243732849756876
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,fp8,4095,0.5010666847229004
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,float16,8191,0.27374933163324994
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,fp8,8191,1.066325346628825
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,1,0.020138667275508244
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,float16,16383,0.8205653031667074
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,1,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,3,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,64,0,1,float16,fp8,16383,3.1976213455200195
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,3,0.019968000551064808
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,7,0.00933333362142245
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,15,0.009759999811649323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,7,0.02096533278624217
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,15,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,31,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,31,0.029696000119050343
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,63,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,63,0.029696000119050343
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,127,0.030720000465710957
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,255,0.03839999934037527
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,127,0.011061333119869232
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,255,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,511,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,511,0.06894933183987935
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,1023,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,1023,0.1304159959157308
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,2047,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,2047,0.25412267446517944
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,4095,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,8191,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,4095,0.501418670018514
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,8191,1.0670080184936523
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,16383,0.06929066777229309
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,1,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,3,0.007631999750932057
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,3,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,16383,3.201706568400065
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,15,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,15,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,31,0.007482666522264481
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,31,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,63,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,255,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,255,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,511,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,511,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,1023,0.018090666582187016
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,1023,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,2047,0.02679466704527537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,2047,0.024746666351954143
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,4095,0.04095999896526337
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,4095,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,8191,0.05751466751098633
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,8191,0.07372800012429555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,16383,0.13943466544151306
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,16383,0.20497065782546997
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,float16,1,0.6309546629587809
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,fp8,1,1.9659093221028645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,float16,3,0.6299146811167399
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,fp8,3,1.968448003133138
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,float16,7,0.6538240114847819
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,float16,15,0.689466635386149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,fp8,7,2.096127986907959
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,fp8,15,2.1789013544718423
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,float16,31,0.8794453144073486
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,float16,63,0.8876533508300781
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,fp8,31,2.906794548034668
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,2,64,0,1,float16,float16,1,0.6355626583099365
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,fp8,63,2.9132798512776694
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,2,64,0,1,float16,float16,3,0.6333599885304769
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,2,64,0,1,float16,fp8,1,1.97872527440389
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,2,64,0,1,float16,float16,7,0.654309352238973
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,2,64,0,1,float16,fp8,3,1.9800747235616047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,2,64,0,1,float16,float16,15,0.6895519892374674
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,2,64,0,1,float16,fp8,7,2.1055146853129068
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,2,64,0,1,float16,fp8,15,2.197162628173828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,2,64,0,1,float16,float16,31,0.8813227017720541
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,2,64,0,1,float16,float16,63,0.8872959613800049
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,float16,1,0.13550933202107748
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,2,64,0,1,float16,fp8,31,2.9088427225748696
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,2,64,0,1,float16,fp8,63,2.910890579223633
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,fp8,1,0.13875200351079306
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,float16,3,0.13550933202107748
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,fp8,3,0.13823999961217245
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,float16,7,0.13567999998728433
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,fp8,7,0.1384106675783793
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,float16,15,0.13550933202107748
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,fp8,15,0.13806933164596558
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,float16,31,0.13550933202107748
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,fp8,31,0.1384106675783793
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,float16,63,0.13550933202107748
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,fp8,63,0.1384106675783793
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,float16,1,0.19438934326171875
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,fp8,1,0.19660800695419312
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,float16,3,0.19438934326171875
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,fp8,3,0.1962666710217794
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,float16,7,0.19473065932591757
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,fp8,7,0.19660800695419312
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,float16,15,0.19490132729212442
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,fp8,15,0.1962666710217794
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,float16,31,0.19421867529551187
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,fp8,31,0.1967786749204
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,float16,63,0.21009065707524618
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,fp8,63,0.1971199909845988
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,float16,1,1.25764266649882
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,float16,3,1.2550880114237468
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,fp8,1,3.929941177368164
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,float16,7,1.3057706356048584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,fp8,3,3.929823875427246
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,float16,15,1.3757333755493164
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,fp8,7,4.195157368977864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,fp8,15,4.366559982299805
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,float16,31,1.7551360130310059
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,2,64,0,1,float16,float16,1,1.2678453127543132
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,fp8,31,5.808298746744792
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,2,64,0,1,float16,float16,3,1.2622506618499756
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,2,64,0,1,float16,fp8,1,3.9582719802856445
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,2,64,0,1,float16,fp8,3,3.9562241236368814
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,2,64,0,1,float16,float16,7,1.3035786946614583
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,2,64,0,1,float16,fp8,7,4.216970761617024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,2,64,0,1,float16,float16,15,1.375381310780843
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,2,64,0,1,float16,float16,31,1.7561813990275066
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,2,64,0,1,float16,fp8,15,4.407295862833659
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,float16,1,0.2676053245862325
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,2,64,0,1,float16,fp8,31,5.810117085774739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,float16,3,0.2676053245862325
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,fp8,1,0.2706773281097412
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,fp8,3,0.2698240081469218
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,float16,7,0.27033599217732746
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,fp8,15,0.27050666014353436
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,fp8,7,0.27084799607594806
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,float16,15,0.27050666014353436
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,float16,31,0.28569600979487103
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,fp8,31,0.2727253238360087
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,float16,1,0.38758401075998944
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,fp8,1,0.3949226538340251
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,float16,3,0.3887786865234375
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,fp8,3,0.3911679983139038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,float16,7,0.3940693140029907
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,fp8,7,0.3947519858678182
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,1,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,float16,15,0.4107946554819743
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,float16,31,0.42444801330566406
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,fp8,15,0.40123732884724933
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,fp8,31,0.4334933360417684
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,1,0.0363520011305809
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,3,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,3,0.0363520011305809
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,7,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,7,0.037717332442601524
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,15,0.01414399966597557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,15,0.038912000755469
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,31,0.017237332959969837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,31,0.05171200136343638
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,63,0.017407999684413273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,63,0.051541333397229515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,127,0.01791999985774358
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,127,0.051541333397229515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,255,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,255,0.06570666531721751
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,511,0.037717332442601524
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,511,0.12032000223795573
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,1023,0.06621866424878438
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,1023,0.22940266132354736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,2047,0.12034133076667786
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,2047,0.4495360056559245
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,4095,0.2486613392829895
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,4095,0.8939519723256429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,8191,0.535210649172465
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,float16,1,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,fp8,1,0.0363520011305809
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,8191,1.9647146860758464
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,float16,3,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,fp8,3,0.03618133316437403
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,float16,7,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,fp8,7,0.03789333254098892
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,float16,15,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,fp8,15,0.040106666584809623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,float16,31,0.01695999999841054
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,fp8,31,0.05171200136343638
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,float16,63,0.017237332959969837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,fp8,63,0.05171200136343638
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,float16,127,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,fp8,127,0.05171200136343638
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,float16,255,0.023381332556406658
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,fp8,255,0.06604266663392384
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,fp8,511,0.12066133817036946
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,float16,511,0.03839466720819473
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,float16,1023,0.06604266663392384
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,fp8,1023,0.2295466661453247
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,float16,2047,0.12117333213488261
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,fp8,2047,0.4500480095545451
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,float16,4095,0.2681173284848531
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,fp8,4095,0.8944479624430338
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,float16,8191,0.5660906632741293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,1,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,64,0,1,float16,fp8,8191,1.9624959627787273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,3,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,3,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,7,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,15,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,31,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,63,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,63,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,255,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,255,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,511,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,511,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,1023,0.020992000897725422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,1023,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,2047,0.032085334261258446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,2047,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,4095,0.048810665806134544
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,4095,0.04113066693147024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,8191,0.06809600194295247
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,8191,0.07406933108965556
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,1,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,1,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,3,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,7,0.008176000167926153
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,3,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,31,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,31,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,63,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,63,0.008010666817426682
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,127,0.03703466554482778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,255,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,511,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,511,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,1023,0.02935466667016347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,2047,0.040106666584809623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,1023,0.01621333385507266
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,2047,0.024746666351954143
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,4095,0.05734399954477946
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,4095,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,8191,0.2053119937578837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,8191,0.07406933108965556
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,1,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,1,0.06996800005435944
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,3,0.07389866809050243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,3,0.025429333249727886
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,7,0.025600001215934753
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,7,0.07577600081761678
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,15,0.031744000812371574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,15,0.09915733337402344
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,31,0.032074667513370514
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,31,0.09898666540781657
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,63,0.03278933217128118
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,63,0.09885332981745402
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,127,0.04215466479460398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,127,0.12680533528327942
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,255,0.07099733253320058
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,255,0.23449599742889404
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,511,0.12800000111262003
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,511,0.4510720173517863
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,1023,0.23944532871246338
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,1023,0.8879786332448324
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,2047,0.4643839995066325
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,2047,1.7640105883280437
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,4095,1.1055786609649658
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,float16,1,0.024405332903067272
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,4095,3.5145387649536133
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,fp8,1,0.07048533360163371
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,float16,3,0.025258667767047882
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,fp8,3,0.07372800012429555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,float16,7,0.025770666698614757
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,fp8,7,0.07645866771539052
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,float16,15,0.031744000812371574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,fp8,15,0.0995253324508667
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,fp8,31,0.09984532992045085
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,float16,31,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,float16,63,0.03293866664171219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,float16,127,0.043178667624791466
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,fp8,63,0.09950400392214458
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,fp8,127,0.12748799721399942
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,float16,255,0.07456533114115398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,fp8,255,0.23552000522613525
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,float16,511,0.13158399860064188
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,fp8,511,0.45127467314402264
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,float16,1023,0.24524799982706705
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,fp8,1023,0.8872959613800049
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,float16,2047,0.4787199894587199
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,fp8,2047,1.7638826370239258
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,float16,4095,1.145685354868571
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,2,128,0,1,float16,fp8,4095,3.520170529683431
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,1,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,1,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,3,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,3,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,7,0.009546666716535887
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,7,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,15,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,15,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,63,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,63,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,127,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,127,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,255,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,255,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,511,0.026965332527955372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,511,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,1023,0.043007999658584595
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,1023,0.0266239990790685
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,31,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,31,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,2047,0.06434133152167003
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,2047,0.04437333345413208
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,4095,0.23176532983779907
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,4095,0.077824001510938
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,1,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,1,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,3,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,3,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,7,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,7,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,15,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,15,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,31,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,31,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,63,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,63,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,127,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,255,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,255,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,511,0.03703466554482778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,511,0.027818667391935985
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,1023,0.053247998158137
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,1023,0.04334933559099833
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,2047,0.07816533247629802
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,2047,0.2218666672706604
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,1,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,3,0.007626666376988093
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,3,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,7,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,7,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,15,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,31,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,15,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,31,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,63,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,63,0.014842666685581207
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,4095,0.43144532044728595
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,4095,0.21964800357818604
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,127,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,127,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,255,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,255,0.03242666771014532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,511,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,511,0.04351999859015147
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,1023,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,1023,0.07169066866238911
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,2047,0.03722666700681051
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,2047,0.11587733030319214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,4095,0.058880001306533813
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,4095,0.21742933988571167
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,8191,0.10803199807802837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,8191,0.37563733259836835
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,16383,0.2034346659978231
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,16383,0.7287466526031494
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,float16,1,0.007477333148320516
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,fp8,1,0.012266666938861212
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,fp8,3,0.012282667060693106
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,float16,7,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,fp8,7,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,float16,15,0.007823999971151352
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,fp8,15,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,float16,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,fp8,31,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,float16,63,0.00850133349498113
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,fp8,63,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,float16,127,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,fp8,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,float16,255,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,fp8,255,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,float16,511,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,fp8,511,0.04607999821503957
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,float16,1023,0.02610666553179423
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,fp8,1023,0.06604800124963124
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,float16,2047,0.037717332442601524
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,float16,4095,0.05870933334032694
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,fp8,2047,0.11673600474993388
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,fp8,4095,0.19882667064666748
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,fp8,8191,0.3752959966659546
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,float16,8191,0.10188800096511841
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,float16,16383,0.1904426614443461
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,2,128,0,1,float16,fp8,16383,0.7282346884409586
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,1,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,7,0.009530666594703993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,15,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,63,0.008485333373149237
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,1,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,3,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,7,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,127,0.008512000242869059
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,127,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,15,0.007653333246707916
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,31,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,255,0.00919999989370505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,31,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,255,0.034128000338872276
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,511,0.04454400142033895
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,511,0.020453333854675293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,1023,0.02628266563018163
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,1023,0.06656000018119812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,2047,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,2047,0.1109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,4095,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,4095,0.19950934251149496
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,8191,0.019626667102177937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,8191,0.3760319948196411
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,16383,0.02218666672706604
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,16383,0.7285760243733724
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,1,0.007381333038210869
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,1,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,3,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,7,0.007370666911204656
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,15,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,31,0.008143999924262365
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,31,0.014495999862750372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,63,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,63,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,127,0.008527999743819237
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,255,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,127,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,255,0.032431999842325844
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,511,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,511,0.04778666794300079
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,1023,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,15,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,7,0.012618667135636011
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,1023,0.06668266654014587
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,2047,0.11040533582369487
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,4095,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,4095,0.1996799906094869
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,8191,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,8191,0.37702401479085285
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,16383,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,16383,0.7288640340169271
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,1,0.006677333265542984
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,1,0.01911466692884763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,3,0.006826666494210561
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,3,0.01911466692884763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,7,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,2047,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,15,0.007685333490371704
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,15,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,31,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,31,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,63,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,63,0.024405332903067272
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,127,0.0086666668454806
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,127,0.024432001014550526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,255,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,255,0.03276266654332479
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,511,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,511,0.05787733197212219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,1023,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,1023,0.09966933727264404
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,2047,0.0435146689414978
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,2047,0.18039466937383017
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,4095,0.08089600006739299
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,4095,0.3534506559371948
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,8191,0.16180800398190817
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,8191,0.6975146929423014
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,16383,0.401589314142863
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,16383,1.3847893079121907
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,float16,1,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,fp8,1,0.019968000551064808
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,float16,3,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,fp8,3,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,float16,7,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,fp8,7,0.020138667275508244
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,float16,15,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,fp8,15,0.021514666577180225
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,float16,31,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,fp8,31,0.025258667767047882
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,float16,63,0.007967999825874964
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,fp8,63,0.025600001215934753
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,fp8,127,0.026965332527955372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,float16,255,0.010058666889866194
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,fp8,255,0.0315733328461647
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,float16,511,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,fp8,511,0.057002668579419456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,float16,1023,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,fp8,1023,0.10717866818110149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,float16,2047,0.045226668318112694
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,fp8,2047,0.18107734123865762
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,float16,4095,0.08345599969228108
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,fp8,4095,0.3537919918696086
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,float16,8191,0.16366933782895407
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,fp8,8191,0.6976479689280192
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,float16,16383,0.4045333464940389
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,1,0.006826666494210561
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,2,128,0,1,float16,fp8,16383,1.3839359283447266
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,1,0.020138667275508244
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,3,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,7,0.007130666946371396
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,7,0.020138667275508244
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,15,0.007653333246707916
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,15,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,31,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,31,0.025258667767047882
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,63,0.009173333023985228
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,63,0.025445332129796345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,127,0.008837333569924036
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,127,0.02628266563018163
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,255,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,255,0.03328000009059906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,511,0.015029333531856537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,511,0.057855998476346336
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,1023,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,1023,0.09574400385220845
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,2047,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,2047,0.1812480092048645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,4095,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,4095,0.35396265983581543
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,8191,0.026965332527955372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,8191,0.6980266571044922
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,16383,0.03362133353948593
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,1,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,16383,1.3854719797770183
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,1,0.02027200038234393
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,3,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,7,0.020992000897725422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,15,0.007797333101431529
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,15,0.021546666820844013
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,31,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,63,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,31,0.025429333249727886
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,63,0.02611200014750163
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,127,0.008816000074148178
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,127,0.025941332181294758
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,255,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,255,0.031061333914597828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,511,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,3,0.020330666253964107
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,7,0.006981333096822103
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,511,0.05239466826121012
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,1023,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,1023,0.09489066402117412
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,2047,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,2047,0.1811306675275167
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,4095,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,8191,0.031061333914597828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,4095,0.3539786736170451
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,8191,0.6981973648071289
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,16383,0.04488533238569895
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,1,0.04437333345413208
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,1,0.12970133622487387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,3,0.04593066871166229
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,16383,1.3842825889587402
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,3,0.13774933417638144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,7,0.04810666541258494
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,7,0.1443839967250824
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,15,0.05989866455396017
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,15,0.192848006884257
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,31,0.06007466713587443
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,63,0.061610668897628784
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,31,0.19182932376861572
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,63,0.19643733898798624
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,127,0.07918933530648549
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,127,0.24780799945195517
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,255,0.13175466656684875
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,255,0.4614826838175456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,511,0.24984000126520792
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,1023,0.47394665082295734
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,511,0.8907093207041422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,2047,0.936789353688558
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,1023,1.7553067207336426
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,float16,1,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,2047,3.4938879013061523
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,fp8,1,0.13107200463612875
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,float16,3,0.045909335215886436
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,fp8,3,0.1367039978504181
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,float16,7,0.04795733094215393
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,fp8,7,0.1431893308957418
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,float16,15,0.06007466713587443
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,fp8,15,0.19184533754984537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,float16,31,0.0602453351020813
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,fp8,31,0.1919999917348226
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,float16,63,0.06126933296521505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,fp8,63,0.19234132766723633
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,float16,127,0.08104533453782399
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,fp8,127,0.2481493353843689
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,float16,255,0.1365120013554891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,fp8,255,0.46062934398651123
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,float16,511,0.25224532683690387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,float16,1023,0.4922026793162028
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,fp8,511,0.8911999861399332
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,float16,2047,0.9828693072001139
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,fp8,1023,1.7554772694905598
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,1,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,fp8,1,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,2,128,0,1,float16,fp8,2047,3.493525187174479
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,3,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,fp8,3,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,7,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,fp8,7,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,15,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,fp8,15,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,31,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,fp8,31,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,63,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,fp8,63,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,127,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,fp8,127,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,255,0.026965332527955372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,fp8,255,0.022015998760859173
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,511,0.04369066655635834
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,fp8,511,0.0315733328461647
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,1023,0.0631466656923294
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,fp8,1023,0.04898133377234141
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,2047,0.23432532946268717
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,fp8,2047,0.08328533172607422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,1,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,1,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,3,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,3,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,7,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,7,0.021333334346612293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,15,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,15,0.022015998760859173
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,31,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,31,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,63,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,63,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,127,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,127,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,255,0.03839999934037527
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,255,0.031061333914597828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,511,0.055125330885251365
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,511,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,1023,0.222378671169281
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,1023,0.08328533172607422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,2047,0.4333226680755615
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,2047,0.2244266668955485
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,1,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,1,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,3,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,3,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,7,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,7,0.028330666323502857
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,15,0.010213333492477735
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,15,0.03054933249950409
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,31,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,31,0.03571200122435888
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,63,0.011946666985750198
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,63,0.03601066768169403
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,127,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,127,0.037205333511034645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,255,0.04510400195916494
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,255,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,511,0.027136000494162243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,511,0.07830933233102162
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,1023,0.04609066744645437
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,2047,0.08584533135096233
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,1023,0.16011200348536173
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,2047,0.2698240081469218
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,4095,0.16503467162450156
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,4095,0.5288960138956705
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,8191,0.32733867565790814
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,8191,1.0450293223063152
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,16383,0.6341973145802816
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,float16,1,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,16383,2.0758186976114907
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,fp8,1,0.02867199977238973
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,float16,3,0.01020800011853377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,fp8,3,0.029370665550231934
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,float16,7,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,fp8,7,0.028501334289709728
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,float16,15,0.010186666622757912
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,fp8,15,0.0308746670683225
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,float16,31,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,fp8,31,0.036864000062147774
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,float16,63,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,fp8,63,0.03703466554482778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,float16,127,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,fp8,127,0.03703466554482778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,float16,255,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,fp8,255,0.04863999783992767
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,float16,511,0.027306665976842243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,float16,1023,0.04642133414745331
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,fp8,511,0.08584533135096233
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,fp8,1023,0.14114133516947427
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,float16,2047,0.08772800366083781
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,fp8,2047,0.2699573238690694
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,float16,4095,0.16964266697565714
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,fp8,4095,0.5287253459294637
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,float16,8191,0.3211946686108907
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,1,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,fp8,8191,1.0451626777648926
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,float16,16383,0.6580906709035238
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,2,128,0,1,float16,fp8,16383,2.0759894053141275
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,1,0.028725333511829376
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,3,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,3,0.02935466667016347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,7,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,7,0.029525332152843475
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,15,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,15,0.031231999397277832
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,31,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,31,0.0363520011305809
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,63,0.011717333147923151
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,63,0.036544000109036766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,127,0.012240000069141388
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,127,0.038058665891488395
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,255,0.015013333410024643
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,255,0.049498667319615684
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,511,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,511,0.07714133461316426
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,1023,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,1023,0.14165332913398743
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,2047,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,2047,0.27050666014353436
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,4095,0.025770666698614757
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,4095,0.5290666818618774
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,8191,0.03754666695992152
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,16383,0.05495466788609823
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,8191,1.0456746419270833
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,1,0.009930666536092758
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,16383,2.0759894053141275
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,1,0.02935466667016347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,3,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,3,0.02918400118748347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,7,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,7,0.03136533250411352
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,15,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,31,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,31,0.03669333209594091
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,63,0.011578666667143503
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,63,0.037205333511034645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,127,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,127,0.036746665835380554
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,255,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,511,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,255,0.045226668318112694
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,511,0.07714133461316426
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,1023,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,1023,0.14147200187047324
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,15,0.0315733328461647
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,2047,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,2047,0.2703146735827128
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,4095,0.0341333324710528
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,4095,0.5290666818618774
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,8191,0.04983466863632202
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,16383,0.22562134265899658
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,8191,1.0465280214945476
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,1,0.08447999755541484
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,1,0.25224532683690387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,16383,2.0758186976114907
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,3,0.0885759989420573
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,3,0.2648746569951375
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,7,0.09198932846387227
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,7,0.2788640062014262
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,15,0.11606933673222859
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,31,0.1160533328851064
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,15,0.37513065338134766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,31,0.37495466073354083
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,63,0.1199679970741272
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,127,0.15269333124160767
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,63,0.37563733259836835
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,127,0.5022720098495483
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,255,0.2513920068740845
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,255,0.9041919708251953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,511,0.47786664962768555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,1023,0.9323519865671793
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,511,1.7488212585449219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,float16,1,0.08447999755541484
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,1023,3.452416102091471
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,fp8,1,0.25309866666793823
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,float16,3,0.08806399504343669
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,fp8,3,0.2658986647923787
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,float16,7,0.09215999643007915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,float16,15,0.11588266491889954
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,fp8,7,0.28040534257888794
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,fp8,15,0.37546666463216144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,float16,31,0.1160533328851064
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,fp8,31,0.37546666463216144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,float16,63,0.11912533640861511
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,float16,127,0.15752533078193665
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,fp8,63,0.37582401434580487
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,fp8,127,0.4856906731923421
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,float16,255,0.2645333409309387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,fp8,255,0.9041919708251953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,float16,511,0.490997314453125
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,fp8,511,1.749333381652832
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,float16,1023,0.9658026695251465
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,1,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,fp8,1,0.028330666323502857
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,3,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,fp8,3,0.02935466667016347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,2,128,0,1,float16,fp8,1023,3.453440030415853
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,7,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,fp8,7,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,15,0.023552000522613525
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,31,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,fp8,15,0.028501334289709728
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,fp8,31,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,63,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,fp8,63,0.02918400118748347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,127,0.027818667391935985
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,fp8,127,0.03276800115903219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,255,0.04625066618124644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,fp8,255,0.03857066730658213
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,511,0.06690133114655812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,fp8,511,0.059562668204307556
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,1,0.031744000812371574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,1023,0.2397866646448771
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,fp8,1023,0.09233066439628601
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,1,0.03942399968703588
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,3,0.031744000812371574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,3,0.03925333420435587
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,7,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,7,0.040106666584809623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,15,0.0314026673634847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,31,0.031231999397277832
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,15,0.040106666584809623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,31,0.040106666584809623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,63,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,63,0.03925333420435587
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,127,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,127,0.04437333345413208
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,255,0.053077335158983864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,255,0.056661332647005715
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,511,0.22801067431767783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,511,0.09557333588600159
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,1023,0.44492801030476886
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,1023,0.2362026572227478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,1,0.1641813317934672
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,fp8,1,0.49987733364105225
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,3,0.17203199863433838
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,fp8,3,0.5266773303349813
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,7,0.1807360053062439
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,fp8,7,0.5517333348592123
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,15,0.2285226583480835
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,fp8,15,0.7430826822916666
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,31,0.22855466604232788
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,fp8,31,0.7439359823862711
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,63,0.23496532440185547
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,fp8,63,0.7441066900889078
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,127,0.30190932750701904
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,fp8,127,0.9630719820658366
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,255,0.4976693391799927
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,2,128,0,1,float16,float16,1,0.1641813317934672
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,2,128,0,1,float16,fp8,1,0.5021013418833414
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,fp8,255,1.797632058461507
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,2,128,0,1,float16,float16,3,0.17169066270192465
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,2,128,0,1,float16,fp8,3,0.5294080177942911
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,2,128,0,1,float16,float16,7,0.1807360053062439
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,2,128,0,1,float16,fp8,7,0.5550080140431722
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,2,128,0,1,float16,float16,15,0.22869332631429037
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,2,128,0,1,float16,float16,31,0.22920533021291098
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,2,128,0,1,float16,fp8,15,0.7435839970906576
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,2,128,0,1,float16,fp8,31,0.7432373364766439
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,2,128,0,1,float16,float16,63,0.23381332556406656
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,2,128,0,1,float16,fp8,63,0.7444907029469808
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,2,128,0,1,float16,float16,127,0.3107840021451314
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,2,128,0,1,float16,fp8,127,0.9635840257008871
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,2,128,0,1,float16,float16,255,0.5159253279368082
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,1,0.04249600072701772
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,fp8,1,0.05085866649945577
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,2,128,0,1,float16,fp8,255,1.8000213305155437
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,3,0.04215466479460398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,fp8,3,0.0506879985332489
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,7,0.04215466479460398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,fp8,7,0.051029334465662636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,15,0.04232533276081085
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,31,0.04215466479460398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,fp8,15,0.05000533163547516
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,fp8,31,0.05085866649945577
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,63,0.04215466479460398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,fp8,63,0.051029334465662636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,127,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,fp8,127,0.056661332647005715
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,255,0.06109866499900818
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,1,0.0554666668176651
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,fp8,255,0.06741333504517873
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,fp8,1,0.07099733253320058
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,3,0.0554666668176651
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,fp8,3,0.07116800049940745
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,7,0.0554666668176651
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,fp8,7,0.07099733253320058
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,15,0.05563733478387197
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,fp8,31,0.07099733253320058
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,31,0.05563733478387197
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,fp8,15,0.07099733253320058
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,63,0.0554666668176651
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,fp8,63,0.07116800049940745
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,127,0.07133866846561432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,fp8,127,0.07987200220425923
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,255,0.23466666539510092
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,fp8,255,0.10222933689753215
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,float16,1,0.32341333230336505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,float16,3,0.3397973378499349
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,fp8,1,0.9936213493347168
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,float16,7,0.35788798332214355
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,fp8,3,1.0513066450754802
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,fp8,7,1.0990933577219646
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,float16,15,0.4534613291422526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,fp8,15,1.481386661529541
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,float16,31,0.45311999320983887
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,fp8,31,1.4786559740702312
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,float16,63,0.4664320151011149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,float16,127,0.593941330909729
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,fp8,63,1.481045405069987
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,fp8,127,1.9182933171590169
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,2,128,0,1,float16,float16,1,0.3237546682357788
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,2,128,0,1,float16,fp8,1,1.000111977259318
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,2,128,0,1,float16,float16,3,0.3394560019175212
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,2,128,0,1,float16,fp8,3,1.0572693347930908
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,2,128,0,1,float16,float16,7,0.35788798332214355
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,2,128,0,1,float16,fp8,7,1.1067732969919841
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,2,128,0,1,float16,float16,15,0.4532906611760457
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,2,128,0,1,float16,fp8,15,1.480533281962077
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,2,128,0,1,float16,float16,31,0.4536319971084595
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,2,128,0,1,float16,float16,63,0.46323732535044354
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,2,128,0,1,float16,fp8,31,1.4817280769348145
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,2,128,0,1,float16,fp8,63,1.4817280769348145
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,2,128,0,1,float16,float16,127,0.6130346854527792
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,float16,1,0.07577600081761678
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,2,128,0,1,float16,fp8,127,1.9181493123372395
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,fp8,1,0.09181867043177287
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,float16,3,0.07594666878382365
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,fp8,3,0.09181867043177287
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,float16,7,0.07560533285140991
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,fp8,7,0.091648002465566
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,float16,15,0.07594666878382365
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,fp8,15,0.09198932846387227
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,float16,31,0.07560533285140991
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,fp8,31,0.091648002465566
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,float16,63,0.07594666878382365
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,fp8,63,0.09147733449935913
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,float16,127,0.1114453375339508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,fp8,127,0.10376532872517903
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,float16,1,0.10359467069307964
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,fp8,1,0.13312000036239624
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,float16,3,0.10359467069307964
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,fp8,3,0.13346133629480997
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,float16,7,0.10359467069307964
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,fp8,7,0.13260799646377563
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,float16,15,0.10376532872517903
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,fp8,15,0.13312000036239624
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,float16,31,0.10359467069307964
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,fp8,31,0.13312000036239624
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,float16,63,0.14472533265749613
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,fp8,63,0.13294933239618936
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,1,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,float16,127,0.25446399052937824
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,fp8,127,0.16776533921559653
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,1,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,3,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,3,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,7,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,7,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,15,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,31,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,15,0.030042665700117748
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,31,0.030213333666324615
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,63,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,63,0.03054933249950409
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,127,0.03806400050719579
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,255,0.021503999829292297
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,511,0.03532800078392029
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,255,0.06911999980608623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,511,0.13056000073750815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,1023,0.0645653357108434
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,1023,0.2558293342590332
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,2047,0.12506133317947388
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,2047,0.5038079818089803
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,4095,0.2573653260866801
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,8191,0.524288018544515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,4095,0.9985706806182861
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,8191,2.1266773541768393
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,float16,1,0.009189333145817121
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,16383,1.595392068227132
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,fp8,1,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,float16,3,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,fp8,3,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,16383,6.349168141682942
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,float16,7,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,fp8,7,0.022848000129063923
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,float16,15,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,fp8,15,0.031061333914597828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,float16,31,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,fp8,31,0.03089066594839096
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,float16,63,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,fp8,63,0.031061333914597828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,float16,127,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,fp8,127,0.03908266623814901
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,float16,255,0.021162666380405426
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,float16,511,0.03669333209594091
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,fp8,255,0.06977599859237671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,fp8,511,0.13226667046546936
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,float16,1023,0.06643733382225037
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,fp8,1023,0.25565866629282635
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,float16,2047,0.12613333264986673
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,fp8,2047,0.5027573506037394
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,float16,4095,0.28489067157109577
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,float16,8191,0.6016266743342081
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,fp8,8191,2.12992000579834
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,fp8,4095,0.9983999729156494
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,float16,16383,1.6481280326843262
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,1,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,1,0.02179733415444692
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,3,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,2,128,0,1,float16,fp8,16383,6.354117075602214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,3,0.02218666672706604
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,7,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,15,0.01090666651725769
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,15,0.0315733328461647
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,7,0.023210667073726654
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,31,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,31,0.031231999397277832
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,63,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,63,0.03143466760714849
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,127,0.03941866755485535
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,255,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,511,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,255,0.07014399766921997
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,511,0.13192533453305563
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,1023,0.01791999985774358
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,1023,0.2563413381576538
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,2047,0.026965332527955372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,2047,0.5038453340530396
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,4095,0.03976533313592275
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,4095,0.9983999729156494
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,8191,0.05870933334032694
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,16383,0.23859200874964395
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,8191,2.1281867027282715
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,16383,6.3648427327473955
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,7,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,7,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,15,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,31,0.008325333396593729
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,31,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,63,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,63,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,255,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,127,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,255,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,511,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,511,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,1023,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,1023,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,2047,0.03515733281771342
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,2047,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,4095,0.07492266595363617
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,4095,0.05717333157857259
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,8191,0.2198186715443929
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,8191,0.13960533340771994
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,16383,0.27613866329193115
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,16383,0.43246932824452716
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,float16,1,0.6420480012893677
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,fp8,1,1.984341303507487
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,float16,7,0.7120213508605957
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,float16,3,0.6748159726460775
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,fp8,7,2.1947786013285318
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,fp8,3,2.1034666697184243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,float16,15,0.9042133490244547
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,fp8,15,2.951338768005371
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,float16,31,0.9028266270955404
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,float16,63,0.9284000396728516
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,fp8,31,2.950655937194824
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,fp8,63,2.9545812606811523
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,2,128,0,1,float16,float16,1,0.6425600051879883
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,2,128,0,1,float16,float16,3,0.6751573085784912
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,2,128,0,1,float16,fp8,1,1.9945813814798992
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,2,128,0,1,float16,fp8,3,2.116096019744873
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,2,128,0,1,float16,float16,7,0.7121919790903727
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,2,128,0,1,float16,float16,15,0.9028213024139404
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,2,128,0,1,float16,fp8,7,2.2099733352661133
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,2,128,0,1,float16,float16,31,0.9033386707305908
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,2,128,0,1,float16,fp8,15,2.9533920288085938
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,2,128,0,1,float16,fp8,31,2.9557758967081704
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,2,128,0,1,float16,float16,63,0.9233067035675049
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,float16,1,0.14660267035166422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,2,128,0,1,float16,fp8,63,2.956085205078125
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,fp8,1,0.176639993985494
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,float16,3,0.14591999848683676
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,fp8,3,0.17612799008687338
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,float16,7,0.14574933052062988
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,fp8,7,0.1764693260192871
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,float16,15,0.15121066570281982
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,fp8,15,0.1764693260192871
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,float16,31,0.187391996383667
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,fp8,31,0.17885865767796835
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,float16,63,0.22220800320307413
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,fp8,63,0.20787199338277182
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,float16,1,0.20667733748753866
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,fp8,1,0.2604373296101888
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,fp8,3,0.2609493335088094
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,float16,3,0.20889600118001303
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,float16,7,0.2126506765683492
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,float16,15,0.24951465924580893
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,fp8,7,0.2616320053736369
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,fp8,15,0.2657279968261719
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,float16,31,0.2984960079193115
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,fp8,31,0.2930346727371216
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,float16,63,0.34201598167419434
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,fp8,63,0.32051199674606323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,float16,1,1.3545494079589844
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,float16,3,1.4148267110188801
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,fp8,1,3.9854294459025064
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,float16,7,1.471999963124593
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,fp8,3,4.2470401128133135
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,fp8,7,4.424362818400065
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,float16,15,1.825424035390218
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,float16,31,1.840127944946289
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,fp8,15,5.910698572794597
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,2,128,0,1,float16,float16,1,1.375914732615153
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,2,128,0,1,float16,float16,3,1.427626609802246
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,fp8,31,5.910869598388672
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,2,128,0,1,float16,fp8,1,4.021589279174805
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,2,128,0,1,float16,float16,7,1.522175947825114
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,2,128,0,1,float16,fp8,3,4.280831972757976
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,2,128,0,1,float16,fp8,7,4.471978823343913
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,2,128,0,1,float16,float16,15,1.8343253135681152
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,2,128,0,1,float16,float16,31,1.8459307352701824
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,2,128,0,1,float16,fp8,15,5.919402440388997
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,float16,1,0.4283733367919922
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,fp8,1,0.49698134263356525
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,float16,3,0.4251306851704915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,2,128,0,1,float16,fp8,31,5.919914881388347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,fp8,3,0.4915200074513753
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,float16,7,0.44646398226420086
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,float16,15,0.4341760079065959
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,fp8,7,0.4935679833094279
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,fp8,15,0.4922026793162028
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,float16,31,0.43212799231211346
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,fp8,31,0.48554666837056476
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,float16,1,0.5614933172861735
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,fp8,1,0.6674773693084717
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,float16,3,0.5524479945500692
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,fp8,3,0.6674773693084717
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,float16,7,0.5672959884007772
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,fp8,7,0.6681599617004395
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,float16,15,0.5785599946975708
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,fp8,15,0.6647466818491617
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,float16,31,0.5867520173390707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,1,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,1,0.037717332442601524
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,3,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,3,0.039077334105968475
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,fp8,31,0.6657706499099731
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,7,0.040106666584809623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,7,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,15,0.017237332959969837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,15,0.05220800141493479
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,31,0.017407999684413273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,31,0.05222400029500326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,63,0.01791999985774358
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,63,0.05205333232879639
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,127,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,127,0.06673066814740498
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,255,0.038058665891488395
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,511,0.06656000018119812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,255,0.12117333213488261
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,511,0.23125332593917847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,1023,0.12168533603350322
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,1023,0.4527786572774251
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,2047,0.23347200949986777
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,2047,0.8978772958119711
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,4095,0.5294080177942911
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,4095,1.78658660252889
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,float16,1,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,fp8,1,0.03788800040880839
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,float16,3,0.013983999689420065
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,8191,1.1008000373840332
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,fp8,3,0.039594667653242745
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,8191,3.9082667032877603
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,float16,7,0.014303999642531076
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,fp8,7,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,fp8,15,0.05239466826121012
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,float16,15,0.017237332959969837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,float16,31,0.01740266631046931
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,float16,63,0.01791999985774358
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,fp8,31,0.05256533126036326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,fp8,63,0.05256533126036326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,float16,127,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,fp8,127,0.06673066814740498
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,float16,255,0.03908266623814901
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,fp8,255,0.12168533603350322
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,float16,511,0.06741333504517873
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,fp8,511,0.2315946618715922
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,float16,1023,0.12526933352152506
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,fp8,1023,0.452949325243632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,float16,2047,0.23893332481384277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,fp8,2047,0.897706667582194
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,float16,4095,0.5708800156911215
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,float16,8191,1.210879961649577
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,fp8,4095,1.7865386009216309
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,1,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,1,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,3,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,2,128,0,1,float16,fp8,8191,3.9075838724772134
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,3,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,7,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,15,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,15,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,31,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,31,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,63,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,63,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,127,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,127,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,255,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,255,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,511,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,1023,0.02611200014750163
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,511,0.0170666662355264
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,1023,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,2047,0.04113066693147024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,2047,0.04334933559099833
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,4095,0.06621866424878438
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,4095,0.07611733178297679
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,8191,0.23142399390538534
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,1,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,8191,0.14148267110188803
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,3,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,3,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,7,0.0084906667470932
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,7,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,31,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,15,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,63,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,63,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,255,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,255,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,511,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,511,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,1023,0.0363520011305809
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,1023,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,2047,0.0554666668176651
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,2047,0.04215466479460398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,4095,0.22050132354100546
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,4095,0.07492266595363617
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,8191,0.42973868052164715
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,8191,0.21555199225743613
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,1,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,1,0.045040001471837364
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,3,0.016895999511082966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,3,0.04508799811204275
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,7,0.017237332959969837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,7,0.04710400104522705
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,15,0.017935999979575474
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,15,0.05034666756788889
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,31,0.022015998760859173
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,31,0.06775466601053874
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,63,0.06534933547178905
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,63,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,127,0.023007998863856
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,127,0.06775466601053874
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,255,0.08703999718030293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,255,0.03071466585000356
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,511,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,511,0.16110933820406595
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,1023,0.08769067128499348
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,1023,0.3089066743850708
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,2047,0.15923200050989786
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,2047,0.6072320143381754
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,4095,0.339626669883728
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,8191,0.7299413681030273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,4095,1.2071306705474854
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,8191,2.650815963745117
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,1,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,1,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,3,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,3,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,7,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,15,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,15,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,31,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,31,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,63,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,255,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,255,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,511,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,511,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,1023,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,1023,0.021162666380405426
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,2047,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,2047,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,4095,0.04147200038035711
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,4095,0.04966400067011515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,8191,0.06843733290831248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,1,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,1,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,8191,0.07406933108965556
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,7,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,15,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,31,0.008522666369875273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,31,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,63,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,127,0.008527999743819237
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,255,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,511,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,511,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,1023,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,1023,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,2047,0.040448000033696495
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,4095,0.057002668579419456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,2047,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,4095,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,8191,0.2063360015551249
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,8191,0.07423999905586243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,1,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,1,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,3,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,15,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,31,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,3,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,7,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,7,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,15,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,31,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,63,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,127,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,127,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,511,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,1023,0.03925333420435587
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,2047,0.04334933559099833
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,4095,0.20377600193023682
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,63,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,255,0.018944000204404194
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,255,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,511,0.02935466667016347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,4095,0.07577600081761678
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,8191,0.3964586655298869
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,2047,0.05751466751098633
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,1,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,1023,0.025770666698614757
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,8191,0.19575466712315878
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,15,0.011274666835864386
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,15,0.007322666545708974
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,31,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,31,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,63,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,63,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,127,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,255,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,255,0.014645333091417948
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,511,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,511,0.017237332959969837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,1023,0.012778667112191519
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,1023,0.03188266605138779
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,2047,0.022357332209746044
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,2047,0.04966400067011515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,4095,0.03299200038115183
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,4095,0.07678933441638947
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,8191,0.04761599997679392
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,8191,0.13499733805656433
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,16383,0.07884799937407176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,16383,0.24985599517822266
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,32767,0.14114133516947427
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,1,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,32767,0.48415998617808026
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,3,0.007653333246707916
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,3,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,7,0.00731733317176501
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,7,0.010970667004585266
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,15,0.011296000331640244
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,31,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,31,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,63,0.0081386665503184
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,63,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,127,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,255,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,255,0.014357333381970724
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,511,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,511,0.018944000204404194
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,1023,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,1023,0.032255999743938446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,2047,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,2047,0.04933866858482361
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,4095,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,4095,0.07526400188604991
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,8191,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,8191,0.13550933202107748
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,16383,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,16383,0.24934399127960205
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,32767,0.0194560003777345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,32767,0.4838666518529256
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,1,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,3,0.011071999867757162
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,15,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,31,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,15,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,31,0.012261333564917246
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,63,0.007807999849319458
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,63,0.013679999858140945
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,127,0.013466666142145792
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,255,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,255,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,511,0.008992000172535578
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,511,0.01791999985774358
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,1023,0.012815999488035837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,1023,0.03416533271471659
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,2047,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,4095,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,2047,0.048469334840774536
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,4095,0.07554133236408234
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,8191,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,8191,0.1353386640548706
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,16383,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,16383,0.24985599517822266
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,32767,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,32767,0.4848639965057373
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,3,0.012634667257467905
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,3,0.00956266683836778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,7,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,15,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,15,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,63,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,63,0.013781332721312841
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,127,0.008527999743819237
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,127,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,255,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,255,0.01482133318980535
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,511,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,511,0.018432000031073887
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,1023,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,1023,0.034815999368826546
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,2047,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,31,0.007658666620651881
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,31,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,4095,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,2047,0.04863999783992767
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,4095,0.07548800110816956
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,8191,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,8191,0.1358506679534912
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,16383,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,16383,0.25014932950337726
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,32767,0.03328000009059906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,32767,0.48452266057332355
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,1,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,3,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,7,0.007333333293596904
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,7,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,15,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,15,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,31,0.0075626665105422335
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,31,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,63,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,63,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,127,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,255,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,255,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,511,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,511,0.03259733319282532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,1023,0.021333334346612293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,1023,0.04454400142033895
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,2047,0.027322667340437572
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,2047,0.06519466638565063
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,4095,0.041984001795450844
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,8191,0.06843733290831248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,4095,0.1083733340104421
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,8191,0.19538666804631552
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,16383,0.11912533640861511
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,16383,0.36950401465098065
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,32767,0.2167466680208842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,1,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,32767,0.718506654103597
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,1,0.011114666859308878
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,7,0.007184000064929326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,7,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,15,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,15,0.011946666985750198
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,31,0.00749333327015241
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,31,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,63,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,63,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,127,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,127,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,255,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,255,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,511,0.032618666688601174
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,511,0.009546666716535887
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,1023,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,1023,0.04744533201058706
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,2047,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,2047,0.06877866884072621
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,4095,0.011946666985750198
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,4095,0.10905599594116211
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,8191,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,8191,0.19574934244155884
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,16383,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,16383,0.36932265758514404
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,32767,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,32767,0.7188479900360107
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,3,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,3,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,7,0.011258666714032492
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,15,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,31,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,31,0.011829332758982977
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,63,0.008357333640257517
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,63,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,255,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,255,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,511,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,511,0.034474665919939675
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,1023,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,1023,0.04390933116277059
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,2047,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,2047,0.06502399841944377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,4095,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,4095,0.10854400197664897
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,8191,0.017237332959969837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,8191,0.19596266746520996
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,16383,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,16383,0.3700053294499715
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,1,0.00720000018676122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,32767,0.03379199902216593
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,32767,0.7200427055358887
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,3,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,7,0.007466666400432587
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,7,0.010890666395425797
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,15,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,31,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,31,0.011829332758982977
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,63,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,127,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,255,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,511,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,255,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,1023,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,511,0.03328000009059906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,2047,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,1023,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,2047,0.0653653343518575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,4095,0.015872000406185787
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,4095,0.10871466994285583
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,8191,0.023552000522613525
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,8191,0.19643733898798624
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,16383,0.0341333324710528
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,16383,0.370469331741333
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,32767,0.054101333022117615
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,32767,0.719871997833252
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,1,0.03054400036732356
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,1,0.08635733524958293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,3,0.03054933249950409
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,3,0.08703999718030293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,7,0.03178133318821589
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,7,0.09107733766237895
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,15,0.032629333436489105
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,15,0.09489066402117412
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,31,0.040106666584809623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,31,0.12610133488972983
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,63,0.04046933352947235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,63,0.12714667121569315
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,127,0.042117332418759666
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,127,0.12731732924779257
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,255,0.05529599885145823
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,255,0.1641813317934672
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,511,0.09233066439628601
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,511,0.30481066306432086
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,1023,0.1699840029080709
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,1023,0.5879466533660889
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,2047,0.3258026639620463
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,2047,1.1560479799906414
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,4095,0.6969760258992513
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,4095,2.3026347160339355
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,1,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,3,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,7,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,7,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,15,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,63,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,63,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,127,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,255,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,255,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,511,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,511,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,1023,0.03515733281771342
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,1023,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,2047,0.048469334840774536
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,2047,0.025770666698614757
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,4095,0.06673066814740498
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,4095,0.04215466479460398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,1,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,1,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,3,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,3,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,7,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,7,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,15,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,15,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,31,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,31,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,63,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,63,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,127,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,127,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,255,0.018944000204404194
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,255,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,511,0.030207999050617218
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,511,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,1023,0.03925333420435587
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,1023,0.02679466704527537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,2047,0.05734399954477946
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,2047,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,4095,0.20736000935236612
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,1,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,1,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,4095,0.0769706666469574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,3,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,3,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,7,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,7,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,15,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,15,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,31,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,63,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,31,0.017237332959969837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,63,0.0170666662355264
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,127,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,127,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,511,0.032085334261258446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,255,0.03054933249950409
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,511,0.040448000033696495
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,1023,0.05751466751098633
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,1023,0.04744533201058706
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,2047,0.20616533358891806
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,2047,0.08089600006739299
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,1,0.006800000245372455
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,1,0.0194560003777345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,3,0.006826666494210561
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,4095,0.3985066811243693
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,4095,0.19831466674804688
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,3,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,7,0.007050666958093643
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,7,0.0189280000825723
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,15,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,15,0.019968000551064808
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,31,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,31,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,63,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,63,0.023210667073726654
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,127,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,127,0.02426133304834366
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,255,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,511,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,255,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,511,0.03259733319282532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,1023,0.0170666662355264
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,1023,0.056320001681645714
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,2047,0.02918400118748347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,2047,0.10733866691589355
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,4095,0.05274133384227753
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,4095,0.179365336894989
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,8191,0.10188800096511841
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,8191,0.35174401601155597
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,16383,0.22033067544301352
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,16383,0.6951253414154053
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,32767,0.4485280116399129
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,1,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,1,0.006826666494210561
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,3,0.018944000204404194
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,3,0.006506666541099548
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,32767,1.431381384531657
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,7,0.0069866664707660675
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,7,0.018944000204404194
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,15,0.007045333584149678
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,15,0.019626667102177937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,31,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,31,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,63,0.008181333541870117
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,127,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,63,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,127,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,255,0.0258240004380544
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,511,0.01091733326514562
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,511,0.03259733319282532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,1023,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,1023,0.05717333157857259
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,2047,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,2047,0.09454933802286784
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,4095,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,4095,0.17937066157658896
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,8191,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,8191,0.3519146839777629
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,16383,0.025600001215934753
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,16383,0.6952959696451823
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,32767,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,1,0.006826666494210561
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,32767,1.4320640563964844
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,1,0.018944000204404194
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,3,0.006805333619316419
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,3,0.019285333653291065
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,7,0.0069866664707660675
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,7,0.019472000499566395
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,15,0.007173333317041397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,15,0.019626667102177937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,31,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,31,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,63,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,63,0.009039999917149544
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,127,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,127,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,255,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,255,0.025258667767047882
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,511,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,511,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,1023,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,1023,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,2047,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,2047,0.09437867005666097
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,4095,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,4095,0.1800266702969869
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,8191,0.024405332903067272
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,8191,0.3510613441467285
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,16383,0.03601066768169403
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,16383,0.6961493492126465
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,32767,0.055125330885251365
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,1,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,32767,1.4329172770182292
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,1,0.01911466692884763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,3,0.006826666494210561
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,3,0.019285333653291065
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,7,0.0194560003777345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,15,0.006826666494210561
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,15,0.019285333653291065
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,31,0.00749333327015241
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,31,0.021157334248224895
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,63,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,63,0.02454400062561035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,127,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,127,0.025199999411900837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,255,0.0266239990790685
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,511,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,1023,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,511,0.030378667016824085
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,1023,0.051407997806866966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,2047,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,2047,0.09454933802286784
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,4095,0.025258667767047882
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,4095,0.1800533334414164
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,8191,0.037205333511034645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,8191,0.35205864906311035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,16383,0.05870933334032694
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,16383,0.6964906851450602
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,32767,0.20428800582885742
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,1,0.056320001681645714
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,32767,1.4335999488830566
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,1,0.16936000188191733
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,3,0.056320001681645714
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,3,0.16924800475438437
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,7,0.05871999760468801
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,7,0.17864000797271729
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,15,0.06126933296521505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,15,0.1867093245188395
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,31,0.07679999868075053
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,31,0.25088000297546387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,63,0.24832000335057577
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,63,0.07746666669845581
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,127,0.08089600006739299
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,127,0.24883200724919638
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,255,0.105813334385554
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,511,0.1795039971669515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,255,0.32255999247233075
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,511,0.6019573211669922
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,1023,0.32870399951934814
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,1023,1.162069320678711
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,2047,0.632149338722229
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,1,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,2047,2.2920533816019693
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,1,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,3,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,7,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,3,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,7,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,15,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,15,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,31,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,31,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,63,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,63,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,127,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,255,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,255,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,511,0.03566933423280716
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,511,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,1023,0.04863999783992767
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,1023,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,2047,0.06809600194295247
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,2047,0.04693333307902018
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,1,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,1,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,3,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,3,0.01791999985774358
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,7,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,7,0.01826133330663045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,15,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,31,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,15,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,31,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,63,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,63,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,127,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,127,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,255,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,255,0.0315733328461647
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,511,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,511,0.03293866664171219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,1023,0.0576853354771932
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,1023,0.048298666874567665
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,2047,0.21009065707524618
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,2047,0.08191999793052673
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,1,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,1,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,3,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,3,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,7,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,7,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,15,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,15,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,31,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,31,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,63,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,127,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,63,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,127,0.03703466554482778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,255,0.0365226666132609
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,255,0.04369066655635834
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,511,0.05273599922657013
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,511,0.05870933334032694
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,1023,0.20548266172409058
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,1023,0.08891733487447102
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,2047,0.2032639980316162
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,2047,0.4031146764755249
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,1,0.10854400197664897
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,1,0.3319466710090637
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,3,0.10871466994285583
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,3,0.3319200078646342
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,7,0.1129813293615977
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,7,0.35072000821431476
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,15,0.11844266454378764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,31,0.15001599987347922
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,63,0.15172266960144043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,31,0.49032533168792725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,15,0.36693334579467773
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,127,0.15752533078193665
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,63,0.49084266026814777
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,127,0.49186134338378906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,255,0.20599466562271118
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,511,0.3457706769307454
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,255,0.6388053496678671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,511,1.195520003636678
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,1,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,1,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,3,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,3,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,7,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,7,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,15,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,15,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,31,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,31,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,63,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,63,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,127,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,127,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,255,0.02679466704527537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,255,0.03839999934037527
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,511,0.051029334465662636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,1,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,511,0.037205333511034645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,1,0.03089066594839096
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,3,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,3,0.031061333914597828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,7,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,7,0.03089066594839096
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,15,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,15,0.03089066594839096
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,31,0.03089066594839096
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,31,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,63,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,63,0.031061333914597828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,127,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,127,0.038058665891488395
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,255,0.03822933385769526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,255,0.04607999821503957
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,511,0.05358933409055074
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,511,0.06178133189678192
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,1,0.05256533126036326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,1,0.051541333397229515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,3,0.05171200136343638
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,3,0.05256533126036326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,7,0.051541333397229515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,7,0.05239466826121012
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,15,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,31,0.051541333397229515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,31,0.05256533126036326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,15,0.05239466826121012
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,63,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,63,0.05239466826121012
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,127,0.051882664362589516
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,127,0.06673066814740498
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,255,0.06724266707897186
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,255,0.08021333316961925
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,511,0.20770132541656494
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,float16,1,0.21179733673731485
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,511,0.10786133011182149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,fp8,1,0.6604959964752197
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,float16,3,0.2121386726697286
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,fp8,3,0.6608213186264038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,float16,7,0.22124266624450684
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,fp8,7,0.7010986804962158
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,float16,15,0.23312000433603922
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,fp8,15,0.7306773662567139
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,float16,31,0.2961066762606303
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,fp8,31,0.9741653601328532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,float16,63,0.2990079919497172
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,fp8,63,0.9748640060424805
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,float16,127,0.3107946713765462
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,fp8,127,0.9782506624857584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,float16,255,0.40804266929626465
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,fp8,255,1.2687359650929768
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,fp8,1,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,float16,3,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,float16,1,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,float16,7,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,fp8,3,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,fp8,7,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,float16,15,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,fp8,15,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,float16,31,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,fp8,31,0.04147200038035711
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,float16,63,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,fp8,63,0.04113066693147024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,float16,127,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,fp8,127,0.043178667624791466
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,float16,255,0.04659200211366018
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,fp8,255,0.04949333270390829
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,float16,1,0.05358933409055074
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,fp8,1,0.05495466788609823
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,fp8,3,0.05478399991989136
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,float16,7,0.05358933409055074
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,float16,3,0.05358933409055074
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,fp8,7,0.05461333195368449
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,float16,15,0.054272000988324486
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,fp8,15,0.05495466788609823
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,float16,31,0.05358933409055074
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,fp8,31,0.05478399991989136
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,float16,63,0.05341866612434387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,fp8,63,0.05461333195368449
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,float16,127,0.05358933409055074
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,fp8,127,0.06894933183987935
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,float16,255,0.06980266670385997
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,fp8,255,0.08516266942024231
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,fp8,1,0.09864532947540283
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,float16,1,0.09693866968154907
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,float16,3,0.09676800171534221
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,fp8,3,0.0988159974416097
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,fp8,7,0.09915733337402344
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,float16,7,0.09693866968154907
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,float16,15,0.09745066364606221
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,fp8,15,0.09949866930643718
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,float16,31,0.09659733374913533
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,fp8,31,0.0988159974416097
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,float16,63,0.09659733374913533
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,fp8,63,0.09898666540781657
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,float16,127,0.09762133161226909
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,fp8,127,0.12680533528327942
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,float16,255,0.21333332856496176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,fp8,255,0.15377066532770792
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,1,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,1,0.018944000204404194
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,3,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,3,0.018933333456516266
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,7,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,7,0.019626667102177937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,15,0.019626667102177937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,15,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,31,0.011242666592200598
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,31,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,63,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,63,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,127,0.013818666338920593
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,255,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,255,0.028501334289709728
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,511,0.01809599995613098
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,511,0.04983466863632202
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,127,0.023552000522613525
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,1023,0.028522667785485584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,1023,0.0930613378683726
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,2047,0.05358933409055074
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,2047,0.18090667327245077
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,4095,0.10188800096511841
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,4095,0.35601067543029785
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,8191,0.19729065895080566
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,8191,0.7156053384145101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,16383,0.4333226680755615
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,16383,2.1329919497172036
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,32767,0.8594720363616943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,1,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,1,0.01911466692884763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,3,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,7,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,3,0.0194560003777345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,32767,4.413440068562825
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,7,0.019797333826621372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,15,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,15,0.019968000551064808
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,31,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,31,0.023183998962243397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,63,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,63,0.023210667073726654
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,127,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,127,0.023210667073726654
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,255,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,255,0.028517333169778187
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,511,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,511,0.05017599960168203
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,1023,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,2047,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,1023,0.09318400422732036
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,2047,0.18090667327245077
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,4095,0.019626667102177937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,4095,0.35601067543029785
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,8191,0.02867199977238973
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,8191,0.7157759666442871
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,16383,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,16383,2.1353333791097007
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,1,0.0100853331387043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,32767,0.06451199948787689
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,1,0.01911466692884763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,3,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,3,0.01911466692884763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,32767,4.421104113260905
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,7,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,7,0.020138667275508244
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,15,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,15,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,31,0.011055999745925268
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,31,0.023381332556406658
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,63,0.01349866638580958
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,63,0.023039999107519787
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,127,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,127,0.02316266546646754
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,255,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,255,0.02867199977238973
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,511,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,511,0.05000533163547516
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,1023,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,1023,0.09335466225941975
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,2047,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,2047,0.18141865730285645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,4095,0.025429333249727886
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,8191,0.03737599899371465
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,4095,0.356170654296875
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,8191,0.7162880102793375
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,16383,0.05905066430568695
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,32767,0.20770132541656494
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,16383,2.138794740041097
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,1,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,32767,4.425386746724446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,15,0.008853333070874214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,31,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,31,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,63,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,63,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,127,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,511,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,511,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,1023,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,2047,0.02679466704527537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,1023,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,2047,0.024746666351954143
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,4095,0.040789333482583366
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,4095,0.040789333482583366
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,8191,0.07355733215808868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,8191,0.0576853354771932
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,16383,0.20428800582885742
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,16383,0.13926399747530618
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,32767,0.3983360131581624
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,32767,0.27153066794077557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,float16,1,0.4189866781234741
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,fp8,1,1.3173759778340657
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,float16,3,0.41864534219106037
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,fp8,3,1.317034641901652
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,float16,7,0.4384426673253377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,fp8,7,1.4015146891276042
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,float16,15,0.4621653159459432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,fp8,15,1.461077372233073
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,float16,31,0.5894826650619507
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,fp8,31,1.9409653345743816
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,float16,63,0.5939199924468994
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,fp8,63,1.9425279299418132
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,float16,127,0.618837316830953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,fp8,127,1.9474773406982422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,fp8,1,0.07406933108965556
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,float16,1,0.07321600119272868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,float16,3,0.07355733215808868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,fp8,3,0.07423999905586243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,float16,7,0.07355733215808868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,fp8,7,0.07423999905586243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,float16,15,0.07355733215808868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,float16,31,0.07287466526031494
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,fp8,15,0.07423999905586243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,fp8,31,0.07406933108965556
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,float16,63,0.07321600119272868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,fp8,63,0.07389866809050243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,float16,127,0.07389866809050243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,fp8,127,0.0773119976123174
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,float16,1,0.10154666503270467
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,fp8,1,0.10342400272687276
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,float16,3,0.10154666503270467
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,float16,7,0.10171733299891154
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,fp8,3,0.10410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,fp8,7,0.10342400272687276
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,float16,15,0.10154666503270467
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,fp8,15,0.1032533347606659
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,float16,31,0.10154666503270467
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,fp8,31,0.10342400272687276
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,float16,63,0.10154666503270467
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,fp8,63,0.10308266679445903
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,float16,127,0.10547199845314026
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,fp8,127,0.13141333063443503
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,float16,1,0.19063466787338257
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,fp8,1,0.1919999917348226
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,float16,3,0.19080533583958945
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,fp8,3,0.1925119956334432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,float16,7,0.19063466787338257
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,fp8,7,0.19285333156585693
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,float16,15,0.19063466787338257
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,fp8,15,0.19268266359965006
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,float16,31,0.19063466787338257
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,fp8,31,0.19217065970102945
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,float16,63,0.19080533583958945
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,fp8,63,0.19234132766723633
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,float16,127,0.26470400889714557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,fp8,127,0.24712532758712769
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,float16,1,0.8306346734364828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,fp8,1,2.631488005320231
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,float16,3,0.8313173453013102
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,fp8,3,2.630330721537272
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,float16,7,0.8717652956644694
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,fp8,7,2.805247942606608
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,float16,15,0.9212586879730225
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,fp8,15,2.9322239557902017
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,float16,31,1.1733333269755046
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,float16,63,1.1844000021616619
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,fp8,31,3.8750346501668296
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,float16,1,0.13858133554458618
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,fp8,1,0.14045866330464682
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,fp8,63,3.8783998489379883
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,float16,3,0.1389226714769999
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,fp8,3,0.14062933127085367
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,float16,7,0.13858133554458618
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,fp8,7,0.14062933127085367
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,float16,15,0.13858133554458618
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,fp8,15,0.14062933127085367
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,float16,31,0.13943466544151306
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,fp8,31,0.14028799533843994
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,float16,63,0.13926399747530618
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,fp8,63,0.14079999923706055
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,fp8,1,0.20036266247431436
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,float16,1,0.19797333081563315
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,float16,3,0.19831466674804688
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,fp8,3,0.20070399840672812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,float16,7,0.19882667064666748
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,fp8,7,0.19933867454528809
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,fp8,15,0.2001919945081075
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,float16,15,0.19814399878184
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,float16,31,0.19882667064666748
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,float16,63,0.2326186696688334
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,fp8,31,0.19950934251149496
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,float16,1,0.37717334429423016
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,fp8,63,0.20070399840672812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,float16,3,0.37717334429423016
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,fp8,1,0.37751468022664386
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,fp8,3,0.37768534819285077
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,float16,7,0.37700267632802326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,fp8,7,0.37751468022664386
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,float16,15,0.37700267632802326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,fp8,15,0.37802668412526447
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,float16,31,0.39611732959747314
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,fp8,31,0.37802668412526447
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,float16,63,0.42683732509613037
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,fp8,63,0.4398080110549927
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,1,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,1,0.027514666318893433
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,3,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,3,0.027461332579453785
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,7,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,7,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,15,0.010389333590865135
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,31,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,15,0.04915200173854828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,31,0.03811199963092804
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,63,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,63,0.03789333254098892
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,127,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,127,0.03806933263937632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,255,0.04868266483147939
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,255,0.0170666662355264
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,511,0.02867199977238973
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,511,0.08874666690826416
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,1023,0.04710400104522705
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,1023,0.1695893406867981
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,2047,0.08618666728337605
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,2047,0.33211733897527057
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,4095,0.17324799299240112
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,4095,0.6563413143157959
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,8191,0.35225598017374676
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,8191,1.4155413309733074
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,1,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,16383,1.0547146797180176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,1,0.027647999425729115
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,3,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,16383,4.24891726175944
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,3,0.027664000789324444
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,7,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,7,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,15,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,31,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,15,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,31,0.03822933385769526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,63,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,63,0.03857066730658213
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,127,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,127,0.038378665844599404
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,255,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,255,0.04898133377234141
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,511,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,511,0.08861866593360901
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,1023,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,1023,0.169813334941864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,2047,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,2047,0.3322880069414775
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,4095,0.02918400118748347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,4095,0.6570666631062826
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,8191,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,8191,1.415189266204834
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,1,0.007327999919652939
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,16383,0.06911999980608623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,16383,4.253904024759929
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,3,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,31,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,31,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,63,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,63,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,127,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,255,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,255,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,511,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,511,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,1023,0.04232533276081085
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,1023,0.01826133330663045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,2047,0.02679466704527537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,2047,0.024746666351954143
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,4095,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,4095,0.04095999896526337
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,8191,0.057855998476346336
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,8191,0.07389866809050243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,16383,0.20616533358891806
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,16383,0.13960533340771994
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,1,0.0075040000180403394
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,7,0.0074986666440963745
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,31,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,63,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,255,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,511,0.018432000031073887
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,511,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,1023,0.02867199977238973
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,1023,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,2047,0.03976533313592275
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,2047,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,4095,0.08328533172607422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,4095,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,8191,0.2044586737950643
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,8191,0.07389866809050243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,16383,0.3978240092595418
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,1,0.01708799973130226
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,16383,0.19746132691701254
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,1,0.046623999873797096
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,3,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,3,0.04915200173854828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,7,0.018229333062966663
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,7,0.051541333397229515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,15,0.02236266682545344
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,15,0.06931200126806895
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,31,0.02253866692384084
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,31,0.06946133573849995
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,63,0.023391999304294586
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,63,0.07031466563542683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,127,0.030906667311986286
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,127,0.09081600109736125
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,255,0.04983466863632202
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,255,0.16230400403340658
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,511,0.09061333537101746
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,511,0.3112480044364929
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,1023,0.16130666931470236
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,1023,0.612010677655538
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,2047,0.3184640010197957
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,2047,1.211903969446818
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,4095,0.7132213115692139
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,4095,2.414933363596598
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,1,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,8191,1.5039146741231282
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,1,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,3,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,3,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,8191,5.273429234822591
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,7,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,7,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,15,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,31,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,15,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,31,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,63,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,127,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,127,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,255,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,255,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,511,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,511,0.018432000031073887
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,1023,0.02679466704527537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,1023,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,2047,0.04232533276081085
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,2047,0.04334933559099833
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,4095,0.06758399804433186
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,4095,0.07628799974918365
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,8191,0.23227733373641968
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,1,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,8191,0.14165332913398743
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,1,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,3,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,3,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,7,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,7,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,15,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,15,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,63,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,63,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,127,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,255,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,255,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,511,0.023039999107519787
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,511,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,1023,0.03703466554482778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,1023,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,2047,0.05649066468079885
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,2047,0.04232533276081085
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,4095,0.22254933913548788
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,4095,0.07526400188604991
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,8191,0.43195732434590656
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,8191,0.23347200949986777
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,1,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,1,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,3,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,3,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,7,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,7,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,15,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,15,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,31,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,63,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,31,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,63,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,255,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,255,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,511,0.037205333511034645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,1023,0.06980266670385997
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,2047,0.22033067544301352
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,2047,0.07867733140786488
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,4095,0.4304213523864746
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,127,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,127,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,4095,0.21998933951059976
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,8191,0.841386636098226
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,511,0.026965332527955372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,8191,0.4121599992116292
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,1023,0.053077335158983864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,1,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,1,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,3,0.007322666545708974
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,3,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,7,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,15,0.012944000462690989
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,31,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,31,0.014544000228246054
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,63,0.008367999767263731
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,63,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,127,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,255,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,255,0.019285333653291065
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,511,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,511,0.0341333324710528
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,1023,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,1023,0.052560001611709595
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,2047,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,2047,0.08413867155710857
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,4095,0.045941332976023354
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,4095,0.13806933164596558
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,8191,0.07901866734027863
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,8191,0.25408534208933514
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,16383,0.1423520048459371
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,16383,0.4916906754175822
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,32767,0.2650453249613444
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,32767,0.9601439634958903
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,3,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,1,0.012453333785136541
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,7,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,15,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,15,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,31,0.014831999937693277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,63,0.008496000121037165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,31,0.008176000167926153
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,63,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,127,0.008336000144481659
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,127,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,255,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,255,0.019461333751678467
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,511,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,511,0.03537066777547201
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,1023,0.02292266736427943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,2047,0.0314026673634847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,1023,0.04983466863632202
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,2047,0.077824001510938
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,4095,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,4095,0.13875200351079306
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,8191,0.019626667102177937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,8191,0.25467199087142944
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,16383,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,16383,0.4920320113499959
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,32767,0.02679466704527537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,32767,0.960863987604777
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,1,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,3,0.012597333639860153
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,7,0.007322666545708974
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,7,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,15,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,31,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,31,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,127,0.015541333705186844
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,255,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,255,0.019626667102177937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,511,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,511,0.0363520011305809
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,1023,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,1023,0.04970666766166687
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,2047,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,2047,0.07812266548474629
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,4095,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,4095,0.13889599839846292
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,8191,0.01621333385507266
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,8191,0.25565866629282635
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,16383,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,16383,0.4922026793162028
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,32767,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,32767,0.9608533382415771
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,1,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,3,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,7,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,15,0.00749333327015241
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,31,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,127,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,1,0.007173333317041397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,255,0.008709333216150602
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,255,0.019285333653291065
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,511,0.01250133290886879
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,511,0.03583466758330663
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,15,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,1023,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,31,0.008005333443482717
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,1023,0.050842667619387306
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,63,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,2047,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,2047,0.077824001510938
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,4095,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,4095,0.1389226714769999
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,8191,0.022357332209746044
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,8191,0.2558293342590332
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,16383,0.027647999425729115
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,1,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,16383,0.4928853511810303
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,32767,0.04471466441949209
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,32767,0.9615466594696045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,1,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,3,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,3,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,7,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,7,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,15,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,31,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,31,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,63,0.00855466661353906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,63,0.014688000082969666
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,127,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,127,0.015040000279744467
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,255,0.04437333345413208
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,511,0.021503999829292297
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,255,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,511,0.047728002071380615
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,1023,0.02679466704527537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,1023,0.07099733253320058
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,2047,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,2047,0.12134400010108948
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,4095,0.0664160003264745
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,4095,0.19899733861287436
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,8191,0.11673600474993388
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,8191,0.3761386473973592
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,16383,0.2146986722946167
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,16383,0.7296000321706136
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,32767,0.43434667587280273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,1,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,1,0.007360000163316727
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,32767,1.4345866839090984
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,3,0.012453333785136541
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,7,0.00731733317176501
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,7,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,15,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,15,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,31,0.014378666877746582
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,31,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,63,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,63,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,127,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,255,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,255,0.03379199902216593
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,511,0.021509334444999695
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,511,0.0481279989083608
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,1023,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,1023,0.07049599786599477
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,2047,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,2047,0.1104213297367096
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,4095,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,4095,0.19933867454528809
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,8191,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,16383,0.023210667073726654
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,8191,0.37677868207295734
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,16383,0.7296000321706136
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,32767,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,1,0.00766933336853981
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,32767,1.4346240361531575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,1,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,3,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,7,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,15,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,15,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,31,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,31,0.03822933385769526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,63,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,63,0.015552000453074774
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,127,0.00874133345981439
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,127,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,255,0.03498666733503342
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,255,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,511,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,511,0.04525866607824961
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,1023,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,1023,0.067071999112765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,2047,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,2047,0.11231999595959981
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,4095,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,4095,0.20036266247431436
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,8191,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,8191,0.3766560157140096
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,16383,0.027818667391935985
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,16383,0.7304533322652181
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,32767,0.04454400142033895
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,1,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,1,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,32767,1.435306708017985
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,3,0.012618667135636011
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,7,0.007466666400432587
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,7,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,15,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,15,0.01349866638580958
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,31,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,31,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,63,0.014831999937693277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,63,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,127,0.00867733359336853
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,127,0.014666666587193808
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,255,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,255,0.03444266567627589
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,511,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,511,0.045567999283472695
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,1023,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,2047,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,1023,0.06724266707897186
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,2047,0.11127466956774394
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,4095,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,4095,0.20155733823776245
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,8191,0.031061333914597828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,8191,0.3775413433710734
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,16383,0.04471466441949209
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,16383,0.7306239604949951
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,32767,0.22528000672658285
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,1,0.0314026673634847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,1,0.08891733487447102
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,32767,1.4358186721801758
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,3,0.03293866664171219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,3,0.09318400422732036
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,7,0.03348266581694285
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,7,0.0981333355108897
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,15,0.040789333482583366
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,15,0.13038933277130127
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,31,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,31,0.12921599547068277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,63,0.04282666742801666
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,63,0.13193600376447043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,127,0.05597866574923197
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,127,0.1660480002562205
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,255,0.09318400422732036
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,255,0.30719999472300213
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,511,0.16895999511082968
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,511,0.5911893447240194
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,1023,0.3199999928474426
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,1023,1.1641173362731934
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,2047,0.6329919894536337
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,2047,2.314922650655111
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,4095,1.4629546801249187
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,1,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,1,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,3,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,3,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,4095,4.614997227986653
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,7,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,7,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,15,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,15,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,31,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,31,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,63,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,63,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,127,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,127,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,255,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,255,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,511,0.027477333943049114
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,511,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,1023,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,1023,0.0264533335963885
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,2047,0.06570666531721751
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,2047,0.045226668318112694
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,4095,0.23654399315516153
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,4095,0.077824001510938
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,1,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,1,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,3,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,3,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,7,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,7,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,15,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,31,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,15,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,31,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,63,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,63,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,127,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,127,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,255,0.023039999107519787
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,255,0.01826133330663045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,511,0.027818667391935985
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,511,0.037717332442601524
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,1023,0.05341866612434387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,1023,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,2047,0.22220800320307413
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,2047,0.07850666840871175
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,4095,0.4312746524810791
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,1,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,1,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,4095,0.22101332743962607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,3,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,3,0.021162666380405426
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,7,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,7,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,15,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,15,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,31,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,31,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,63,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,127,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,63,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,127,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,255,0.03754666695992152
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,255,0.030378667016824085
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,511,0.05461333195368449
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,511,0.04983466863632202
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,1023,0.21913599967956543
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,1023,0.08294400076071422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,2047,0.42871467272440594
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,2047,0.2228906750679016
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,4095,0.8427519798278809
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,1,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,1,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,4095,0.41727999846140545
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,3,0.007151999821265538
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,3,0.020992000897725422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,7,0.00721066693464915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,7,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,15,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,15,0.022015998760859173
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,31,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,31,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,63,0.025941332181294758
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,127,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,127,0.0264533335963885
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,255,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,255,0.031061333914597828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,511,0.017237332959969837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,511,0.05256533126036326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,1023,0.09545600414276123
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,1023,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,2047,0.18107734123865762
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,2047,0.05273599922657013
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,4095,0.09916266798973083
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,4095,0.35396265983581543
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,8191,0.19831466674804688
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,8191,0.6985387007395426
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,16383,0.43093331654866535
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,16383,1.3854719797770183
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,32767,0.8647680282592773
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,32767,2.863274574279785
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,1,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,7,0.00720000018676122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,3,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,7,0.021333334346612293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,15,0.007685333490371704
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,15,0.022367998957633972
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,31,0.008858666444818178
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,31,0.025413334369659424
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,63,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,63,0.025770666698614757
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,127,0.026965332527955372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,127,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,255,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,255,0.0341333324710528
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,511,0.017237332959969837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,511,0.05256533126036326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,1023,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,1023,0.09523199995358785
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,2047,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,4095,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,2047,0.18141865730285645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,4095,0.3546453317006429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,8191,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,8191,0.6987199783325195
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,16383,0.03498666733503342
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,16383,1.3854613304138184
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,32767,0.05529599885145823
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,1,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,1,0.021327999730904896
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,32767,2.8629334767659507
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,3,0.020992000897725422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,7,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,7,0.021168000996112823
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,15,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,15,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,31,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,31,0.025941332181294758
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,63,0.025600001215934753
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,63,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,127,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,127,0.025807999074459076
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,255,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,511,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,255,0.0314026673634847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,511,0.052704001466433205
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,1023,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,1023,0.09593066573143005
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,2047,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,2047,0.18175999323527017
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,4095,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,4095,0.3546666701634725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,8191,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,8191,0.6987093289693197
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,16383,0.04625066618124644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,16383,1.385642687479655
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,32767,0.22937599817911783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,1,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,1,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,32767,2.8644692103068032
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,3,0.021525333325068157
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,3,0.007146666447321574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,7,0.007151999821265538
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,7,0.022015998760859173
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,15,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,15,0.022650666534900665
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,31,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,31,0.025487999121348064
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,63,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,63,0.025600001215934753
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,127,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,127,0.0269813338915507
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,255,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,255,0.0314026673634847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,511,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,511,0.05273599922657013
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,1023,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,1023,0.09557333588600159
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,2047,0.04505600035190582
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,2047,0.18187199036280313
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,4095,0.03310933212439219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,4095,0.35447466373443604
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,8191,0.05034666756788889
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,8191,0.6993920008341471
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,16383,0.2239146629969279
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,32767,0.43485867977142334
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,16383,1.385983943939209
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,1,0.057855998476346336
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,1,0.17203199863433838
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,3,0.06092800199985504
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,3,0.18056533734003702
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,32767,2.902016003926595
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,7,0.0628053347269694
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,7,0.1889280080795288
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,15,0.07851199805736542
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,15,0.25224532683690387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,31,0.07884799937407176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,31,0.2524159948031108
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,63,0.08106666803359985
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,63,0.2524159948031108
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,127,0.1063253382841746
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,127,0.3259733319282532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,255,0.17612799008687338
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,255,0.6056960026423136
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,511,0.3269973397254944
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,511,1.1717973550160725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,1023,0.6315199931462606
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,2047,1.259178638458252
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,1023,2.308997313181559
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,1,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,3,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,1,0.017407999684413273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,3,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,2047,4.5972747802734375
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,7,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,7,0.017407999684413273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,15,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,31,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,31,0.017237332959969837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,15,0.017407999684413273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,127,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,63,0.017407999684413273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,127,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,255,0.028160000840822857
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,255,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,511,0.03242666771014532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,511,0.04505600035190582
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,1023,0.04898133377234141
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,2047,0.2384213407834371
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,1023,0.06485333542029063
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,2047,0.08413867155710857
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,1,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,1,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,3,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,3,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,7,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,7,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,15,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,15,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,31,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,31,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,63,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,63,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,127,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,127,0.024746666351954143
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,255,0.030378667016824085
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,511,0.05597866574923197
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,255,0.03908266623814901
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,511,0.0506879985332489
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,1023,0.22732800245285034
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,1023,0.08328533172607422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,2047,0.43775999546051025
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,2047,0.22801067431767783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,1,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,1,0.03822933385769526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,3,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,3,0.03822933385769526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,7,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,7,0.03822933385769526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,15,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,31,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,15,0.03822933385769526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,31,0.03822933385769526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,63,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,127,0.03822933385769526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,63,0.038058665891488395
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,127,0.043178667624791466
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,255,0.052906667192777
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,255,0.05461333195368449
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,511,0.22340265909830728
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,511,0.09352533022562663
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,1023,0.23057067394256592
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,1023,0.4374186595280965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,2047,0.8574293454488119
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,1,0.11122666796048482
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,2047,0.4346880118052165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,1,0.3351893424987793
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,3,0.11639466881752014
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,3,0.3535733222961426
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,7,0.12236799796422322
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,15,0.15380799770355225
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,7,0.37085866928100586
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,15,0.4978346824645996
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,31,0.1539413332939148
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,31,0.49766401449839276
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,63,0.15869866808255514
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,63,0.4986773331960042
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,127,0.20770132541656494
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,127,0.6446079810460409
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,255,0.34252798557281494
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,255,1.2024959723154705
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,511,0.6502399841944376
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,1,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,511,2.3272107442220054
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,1,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,3,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,3,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,7,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,7,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,15,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,15,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,31,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,31,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,63,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,63,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,127,0.028330666323502857
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,255,0.039594667653242745
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,255,0.04778666794300079
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,127,0.03310933212439219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,511,0.0682666649421056
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,511,0.059392000238100685
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,1,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,1,0.040106666584809623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,3,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,3,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,7,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,15,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,7,0.040106666584809623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,15,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,31,0.032085334261258446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,31,0.040106666584809623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,63,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,63,0.040106666584809623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,127,0.04437333345413208
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,127,0.040106666584809623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,255,0.057002668579419456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,255,0.054101333022117615
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,511,0.23193599780400595
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,511,0.09489066402117412
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,1,0.05341866612434387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,1,0.06877866884072621
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,3,0.05358933409055074
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,3,0.06894933183987935
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,7,0.05341866612434387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,15,0.05358933409055074
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,7,0.06894933183987935
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,15,0.06894933183987935
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,31,0.05341866612434387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,63,0.05341866612434387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,31,0.06877866884072621
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,63,0.06877866884072621
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,127,0.07765333354473114
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,127,0.06894933183987935
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,255,0.2249386707941691
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,255,0.09966933727264404
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,511,0.44014934698740643
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,511,0.22869332631429037
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,1,0.21794132391611734
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,fp8,1,0.6655999819437662
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,3,0.22869332631429037
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,fp8,3,0.7041172981262207
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,7,0.2404693365097046
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,fp8,7,0.7369386355082194
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,15,0.3037866751352946
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,fp8,15,0.9888746738433838
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,31,0.30429865916570026
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,63,0.31300799051920575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,fp8,31,0.9886720180511475
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,fp8,63,0.9907200336456299
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,127,0.4113066593805949
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,fp8,127,1.2810239791870117
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,255,0.6714026927947998
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,1,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,fp8,255,2.388538678487142
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,fp8,1,0.05341866612434387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,3,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,fp8,3,0.05273599922657013
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,fp8,7,0.05358933409055074
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,7,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,15,0.04369066655635834
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,fp8,15,0.052906667192777
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,31,0.04369066655635834
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,fp8,31,0.053247998158137
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,63,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,fp8,63,0.053077335158983864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,127,0.051882664362589516
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,255,0.06263466676076253
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,fp8,127,0.0602453351020813
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,fp8,255,0.07048533360163371
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,1,0.05717333157857259
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,fp8,1,0.07202133536338806
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,3,0.05717333157857259
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,fp8,3,0.07202133536338806
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,7,0.056832000613212585
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,fp8,7,0.0721919983625412
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,15,0.0576853354771932
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,fp8,15,0.07202133536338806
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,fp8,31,0.07202133536338806
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,31,0.05717333157857259
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,63,0.05836800237496694
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,fp8,63,0.0721919983625412
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,127,0.07372800012429555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,fp8,127,0.08106666803359985
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,255,0.2435413400332133
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,fp8,255,0.10342400272687276
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,1,0.10052266716957092
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,fp8,1,0.1302186648050944
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,3,0.10035199920336406
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,fp8,3,0.1295360028743744
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,7,0.10052266716957092
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,fp8,7,0.1293653349081675
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,31,0.10069333513577779
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,fp8,31,0.12987732887268066
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,fp8,15,0.12987732887268066
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,15,0.10069333513577779
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,63,0.10666666428248088
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,fp8,63,0.12970667084058127
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,127,0.23057067394256592
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,fp8,127,0.15172266960144043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,1,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,255,0.45021867752075195
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,fp8,255,0.24132267634073892
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,1,0.020138667275508244
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,3,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,3,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,7,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,7,0.021333334346612293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,15,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,15,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,31,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,31,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,63,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,63,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,127,0.029696000119050343
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,255,0.01791999985774358
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,255,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,511,0.02867199977238973
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,1023,0.05273599922657013
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,511,0.09386666615804036
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,1023,0.18193066120147705
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,2047,0.10222933689753215
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,2047,0.35703468322753906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,4095,0.1996799906094869
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,4095,0.7069013118743896
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,8191,0.39134931564331055
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,8191,1.4267733891805012
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,16383,0.8485546906789144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,16383,4.2374827067057295
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,1,0.010245333115259806
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,32767,1.6701706250508626
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,1,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,3,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,3,0.021162666380405426
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,7,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,32767,8.681429545084635
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,7,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,15,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,15,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,31,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,31,0.023552000522613525
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,63,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,63,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,127,0.029696000119050343
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,255,0.01791999985774358
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,255,0.05120000243186951
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,511,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,511,0.09404266873995464
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,1023,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,1023,0.1819093426068624
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,2047,0.01791999985774358
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,2047,0.3566506703694661
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,4095,0.025941332181294758
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,4095,0.7066986560821533
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,8191,0.03788800040880839
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,8191,1.4269439379374187
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,16383,0.05597866574923197
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,1,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,1,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,32767,0.23756800095240274
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,3,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,16383,4.241408030192058
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,3,0.021333334346612293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,7,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,32767,8.694784164428711
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,7,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,15,0.011136000355084738
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,31,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,15,0.025072000920772552
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,31,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,63,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,63,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,127,0.014303999642531076
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,127,0.029696000119050343
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,255,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,255,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,1023,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,511,0.0942080020904541
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,511,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,1023,0.18210132916768393
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,2047,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,2047,0.3568640152613322
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,4095,0.03379199902216593
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,4095,0.7079253196716309
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,8191,0.05017599960168203
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,8191,1.427626609802246
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,16383,0.22528000672658285
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,1,0.008522666369875273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,32767,0.43775999546051025
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,1,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,16383,4.24721082051595
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,7,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,32767,9.014101028442383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,15,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,15,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,31,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,63,0.008309333274761835
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,127,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,255,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,255,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,511,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,511,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,1023,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,1023,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,2047,0.034815999368826546
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,2047,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,4095,0.05751466751098633
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,4095,0.07406933108965556
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,8191,0.21794132391611734
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,8191,0.14028799533843994
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,16383,0.43093331654866535
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,16383,0.2768213351567586
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,32767,0.8557226657867432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,32767,0.5379413366317749
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,float16,1,0.43197866280873615
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,fp8,1,1.3282986481984456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,float16,3,0.45207464694976807
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,fp8,3,1.4067893028259277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,float16,7,0.4771840174992879
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,fp8,7,1.4701226552327473
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,float16,15,0.6045013268788656
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,fp8,15,1.9705173174540203
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,float16,31,0.6055253346761068
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,float16,63,0.6208853324254354
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,fp8,31,1.9698559443155925
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,fp8,63,1.9725866317749023
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,float16,127,0.8128853638966879
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,float16,1,0.07935999830563863
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,fp8,1,0.09676800171534221
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,fp8,127,2.5545387268066406
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,float16,3,0.07901866734027863
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,fp8,3,0.09676800171534221
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,float16,7,0.07901866734027863
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,fp8,7,0.09676800171534221
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,float16,15,0.07884799937407176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,fp8,15,0.09676800171534221
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,float16,31,0.07901866734027863
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,float16,63,0.0795306662718455
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,fp8,31,0.09727999567985535
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,fp8,63,0.09710933764775594
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,float16,127,0.16913066307703653
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,float16,1,0.10683733224868774
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,fp8,127,0.11059199770291646
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,fp8,1,0.1353386640548706
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,float16,3,0.10683733224868774
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,fp8,3,0.13550933202107748
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,float16,7,0.10700800021489461
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,fp8,7,0.1353386640548706
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,float16,15,0.10717866818110149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,fp8,15,0.1353386640548706
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,float16,31,0.1083733340104421
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,fp8,31,0.13550933202107748
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,float16,63,0.16895999511082968
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,fp8,63,0.13550933202107748
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,fp8,127,0.19217065970102945
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,float16,1,0.19575466712315878
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,float16,127,0.2769920031229655
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,fp8,1,0.25446399052937824
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,float16,3,0.19592533508936563
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,fp8,3,0.2542933424313863
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,float16,7,0.19575466712315878
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,fp8,7,0.2542933424313863
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,float16,15,0.19780266284942627
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,fp8,15,0.2542933424313863
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,float16,31,0.27357866366704303
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,fp8,31,0.25565866629282635
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,float16,63,0.2993493278821309
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,fp8,63,0.2874026695887248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,float16,127,0.46745598316192627
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,fp8,127,0.3572053511937459
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,float16,1,0.8575999736785889
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,fp8,1,2.653183937072754
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,float16,3,0.9004373550415039
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,float16,7,0.9509546756744385
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,fp8,3,2.8168532053629556
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,fp8,7,2.9392318725585938
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,float16,15,1.204906702041626
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,fp8,15,3.9389867782592773
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,float16,31,1.2052480379740398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,fp8,31,3.932501475016276
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,float16,63,1.256277322769165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,float16,1,0.18312533696492514
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,fp8,63,3.9381281534830728
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,fp8,1,0.22118399540583292
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,float16,3,0.20923733711242676
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,fp8,3,0.21572266022364298
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,float16,7,0.21640533208847046
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,fp8,7,0.22169599930445352
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,float16,15,0.22562134265899658
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,fp8,15,0.2326186696688334
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,float16,31,0.2402986685434977
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,fp8,31,0.24729599555333456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,float16,63,0.27767467498779297
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,fp8,63,0.25702399015426636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,float16,1,0.24883200724919638
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,fp8,1,0.30958932638168335
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,float16,3,0.2609493335088094
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,fp8,3,0.3099306623140971
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,float16,7,0.28484266996383667
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,fp8,7,0.32102400064468384
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,float16,15,0.31487999359766644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,fp8,15,0.32870399951934814
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,float16,31,0.3256319959958394
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,fp8,31,0.3397973378499349
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,fp8,63,0.3553280035654704
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,float16,63,0.3778560161590576
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,fp8,1,0.570026675860087
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,float16,1,0.44817066192626953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,float16,3,0.45875199635823566
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,fp8,3,0.5741226673126221
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,float16,7,0.47035733858744305
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,fp8,7,0.5818026860555013
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,float16,15,0.5138773520787557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,fp8,15,0.5966506799062093
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,float16,31,0.5384533405303955
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,float16,63,0.614741325378418
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,fp8,31,0.6104746659596761
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,1,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,fp8,63,0.6272000074386597
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,1,0.028698667883872986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,3,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,3,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,7,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,7,0.030559999247392017
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,15,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,15,0.03841600070397059
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,31,0.013317332913478216
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,31,0.038912000755469
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,63,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,63,0.03875733415285746
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,127,0.017237332959969837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,127,0.0491946687301
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,255,0.028160000840822857
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,255,0.08925867080688477
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,511,0.04778666794300079
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,511,0.17117865880330405
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,1023,0.08567466338475545
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,1023,0.33501867453257245
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,2047,0.16366933782895407
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,2047,0.6597973505655924
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,4095,0.36164267857869464
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,4095,1.3109013239542644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,8191,0.7067306836446127
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,8191,2.822655995686849
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,16383,2.0990293820699057
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,1,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,1,0.029557332396507263
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,3,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,3,0.030378667016824085
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,7,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,7,0.031061333914597828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,16383,8.452437082926432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,15,0.039594667653242745
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,31,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,15,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,31,0.038912000755469
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,63,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,63,0.03890133400758108
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,127,0.0170666662355264
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,127,0.04947733382383982
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,255,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,255,0.08959999680519104
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,511,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,1023,0.018432000031073887
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,511,0.17099199692408243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,1023,0.33433600266774494
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,2047,0.027306665976842243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,2047,0.6598079999287924
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,4095,0.040618665516376495
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,8191,0.059392000238100685
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,4095,1.3120853106180828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,16383,0.23893332481384277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,8191,2.822826703389486
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,1,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,3,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,16383,8.463018417358398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,3,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,31,0.008336000144481659
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,63,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,127,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,255,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,255,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,511,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,511,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,1023,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,1023,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,2047,0.03515733281771342
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,2047,0.04215466479460398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,4095,0.05751466751098633
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,4095,0.07458133498827617
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,8191,0.2208426594734192
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,8191,0.14045866330464682
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,16383,0.43195732434590656
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,16383,0.2757973273595174
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,3,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,3,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,7,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,15,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,31,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,63,0.009365333244204521
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,127,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,127,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,255,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,255,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,511,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,1023,0.03583999971548716
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,511,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,1023,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,2047,0.0554666668176651
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,2047,0.04215466479460398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,4095,0.22016000747680664
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,4095,0.07492266595363617
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,8191,0.21555199225743613
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,8191,0.42632532119750977
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,16383,0.8456532955169678
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,16383,0.40994131565093994
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,1,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,1,0.027647999425729115
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,3,0.009733333562811216
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,3,0.027477333943049114
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,7,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,7,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,15,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,31,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,15,0.029359998802344005
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,31,0.037717332442601524
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,63,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,63,0.03839999934037527
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,127,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,127,0.03822933385769526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,255,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,255,0.048810665806134544
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,511,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,511,0.08840533097585042
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,1023,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,1023,0.1699999968210856
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,2047,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,2047,0.3326293428738912
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,4095,0.029696000119050343
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,4095,0.6568959951400757
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,8191,0.04471466441949209
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,16383,0.06911999980608623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,8191,1.4153440793355305
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,1,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,16383,4.2491200764973955
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,3,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,7,0.007477333148320516
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,15,0.0074879998962084455
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,31,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,63,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,63,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,127,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,255,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,255,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,511,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,511,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,1023,0.018090666582187016
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,1023,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,2047,0.027136000494162243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,2047,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,4095,0.04113066693147024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,4095,0.04095999896526337
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,8191,0.0576853354771932
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,8191,0.07406933108965556
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,16383,0.20360533396402994
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,16383,0.13960533340771994
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,3,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,7,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,15,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,31,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,31,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,63,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,127,0.008527999743819237
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,255,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,255,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,511,0.018432000031073887
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,1023,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,511,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,1023,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,2047,0.040448000033696495
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,2047,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,4095,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,4095,0.05717333157857259
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,8191,0.22203733523686728
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,8191,0.07389866809050243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,16383,0.3959466616312663
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,16383,0.19421867529551187
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,1,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,3,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,1,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,7,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,3,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,7,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,31,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,15,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,127,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,255,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,511,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,1023,0.03908266623814901
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,31,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,63,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,127,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,63,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,1023,0.025941332181294758
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,2047,0.057002668579419456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,255,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,511,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,2047,0.04334933559099833
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,4095,0.202239990234375
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,4095,0.07611733178297679
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,8191,0.39321601390838623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,8191,0.1925119956334432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,16383,0.7806293169657389
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,16383,0.37495466073354083
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,1,0.011077333241701126
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,3,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,7,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,15,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,15,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,31,0.007333333293596904
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,31,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,63,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,63,0.013493333011865616
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,127,0.013280000537633896
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,255,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,255,0.013946666071812311
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,511,0.00855466661353906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,511,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,1023,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,1023,0.01826133330663045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,2047,0.0129120002190272
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,2047,0.03172266731659571
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,4095,0.022015998760859173
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,4095,0.0576800008614858
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,8191,0.09113599856694539
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,8191,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,16383,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,16383,0.1570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,32767,0.2926933368047078
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,32767,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,1,0.010949333508809408
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,65535,0.023381332556406658
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,65535,0.5758293469746908
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,3,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,7,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,15,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,31,0.007333333293596904
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,31,0.011946666985750198
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,63,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,63,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,127,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,127,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,255,0.00871999996403853
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,255,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,511,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,511,0.017978666971127193
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,2047,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,1023,0.018570666511853535
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,1023,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,2047,0.03272533416748047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,4095,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,4095,0.05390933156013489
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,8191,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,8191,0.09113599856694539
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,16383,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,16383,0.15718400478363037
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,32767,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,32767,0.2927306691805522
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,1,0.007018666714429855
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,1,0.011381333072980246
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,65535,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,65535,0.5765120188395182
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,3,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,15,0.007184000064929326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,15,0.011488000551859537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,31,0.007541333635648091
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,31,0.011744000017642975
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,63,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,63,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,127,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,255,0.014389333625634512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,511,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,511,0.017632000148296356
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,1023,0.01166933278242747
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,1023,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,2047,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,2047,0.03223466624816259
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,4095,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,4095,0.05376000205675761
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,8191,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,8191,0.09149866302808125
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,16383,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,16383,0.15735466281572977
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,1,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,32767,0.024746666351954143
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,32767,0.29335467020670575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,65535,0.577023983001709
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,65535,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,3,0.01108266661564509
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,7,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,15,0.007173333317041397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,31,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,31,0.007327999919652939
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,63,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,63,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,127,0.013712000101804733
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,255,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,255,0.014495999862750372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,3,0.007296000296870868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,511,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,511,0.01964266722400983
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,1023,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,15,0.011551999797423681
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,1023,0.018320000420014065
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,2047,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,2047,0.03239466746648153
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,4095,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,4095,0.05426133175690969
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,8191,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,8191,0.09130666653315227
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,16383,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,16383,0.15718400478363037
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,32767,0.03293866664171219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,32767,0.2935466567675273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,65535,0.05273599922657013
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,1,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,65535,0.5771946509679159
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,3,0.007327999919652939
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,7,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,15,0.007322666545708974
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,31,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,31,0.011946666985750198
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,63,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,63,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,127,0.009269333134094873
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,255,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,255,0.013658666362365087
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,511,0.00890666681031386
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,511,0.018090666582187016
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,1023,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,1023,0.0339626669883728
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,2047,0.02218666672706604
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,2047,0.052906667192777
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,4095,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,4095,0.07560533285140991
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,8191,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,8191,0.14131200313568115
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,16383,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,16383,0.24921600023905435
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,32767,0.019626667102177937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,32767,0.4838399887084961
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,65535,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,1,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,65535,0.719701369603475
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,3,0.0069919998447100324
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,7,0.007322666545708974
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,7,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,15,0.00731733317176501
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,15,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,31,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,31,0.011946666985750198
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,63,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,63,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,127,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,127,0.01434133326013883
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,255,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,255,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,511,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,511,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,1023,0.012650666137536367
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,1023,0.03293866664171219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,2047,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,2047,0.053077335158983864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,4095,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,4095,0.07526400188604991
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,8191,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,8191,0.13550933202107748
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,16383,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,16383,0.24934399127960205
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,32767,0.024746666351954143
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,32767,0.48418132464090985
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,65535,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,1,0.0074879998962084455
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,65535,0.7195306619008383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,3,0.007327999919652939
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,3,0.011792000383138657
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,7,0.007285333548982938
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,7,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,15,0.007290666922926903
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,15,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,31,0.0075519997626543045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,31,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,63,0.01370666672786077
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,63,0.008026666939258575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,127,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,255,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,255,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,511,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,1023,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,511,0.019968000551064808
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,1023,0.034304000437259674
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,2047,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,2047,0.048469334840774536
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,4095,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,4095,0.07594666878382365
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,8191,0.0170666662355264
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,8191,0.1353386640548706
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,16383,0.021503999829292297
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,16383,0.24951465924580893
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,32767,0.03328000009059906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,32767,0.48418132464090985
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,65535,0.05341866612434387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,65535,0.721407969792684
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,3,0.00702400008837382
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,3,0.011077333241701126
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,7,0.007311999797821045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,7,0.011637333780527115
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,15,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,15,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,31,0.007663999994595845
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,31,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,127,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,63,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,63,0.014309333016475042
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,255,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,255,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,511,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,511,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,1023,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,1023,0.032442666590213776
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,2047,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,2047,0.04863999783992767
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,4095,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,4095,0.07577600081761678
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,8191,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,8191,0.13550933202107748
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,16383,0.034815999368826546
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,16383,0.2500266631444295
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,32767,0.054101333022117615
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,32767,0.4847093423207601
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,65535,0.20565332969029745
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,1,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,65535,0.7200906276702881
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,1,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,3,0.008320000022649765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,3,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,7,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,7,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,15,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,31,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,127,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,255,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,255,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,511,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,511,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,1023,0.021162666380405426
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,1023,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,2047,0.032085334261258446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,2047,0.025770666698614757
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,4095,0.04949333270390829
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,4095,0.04215466479460398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,8191,0.06843733290831248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,1,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,8191,0.07492266595363617
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,3,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,7,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,7,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,15,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,15,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,31,0.007834666719039282
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,31,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,63,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,63,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,255,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,511,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,511,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,1023,0.02918400118748347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,1023,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,2047,0.040448000033696495
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,2047,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,4095,0.057002668579419456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,4095,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,1,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,8191,0.20428800582885742
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,8191,0.07406933108965556
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,1,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,3,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,3,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,7,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,7,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,15,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,15,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,31,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,31,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,63,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,63,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,127,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,127,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,255,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,255,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,511,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,511,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,1023,0.03908266623814901
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,1023,0.025941332181294758
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,2047,0.05751466751098633
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,2047,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,4095,0.20360533396402994
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,4095,0.0769706666469574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,1,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,1,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,8191,0.39628799756368
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,8191,0.19336533546447754
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,3,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,3,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,7,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,7,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,15,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,15,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,31,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,31,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,63,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,63,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,127,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,127,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,255,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,255,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,511,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,511,0.031061333914597828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,1023,0.05751466751098633
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,1023,0.046762665112813316
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,2047,0.20309333006540933
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,2047,0.08089600006739299
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,4095,0.3954346577326457
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,4095,0.19729065895080566
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,1,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,8191,0.7831892967224121
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,8191,0.37887998421986896
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,1,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,3,0.00730666642387708
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,7,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,15,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,31,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,31,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,63,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,63,0.013440000514189402
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,255,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,255,0.03328000009059906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,511,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,511,0.03341866781314214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,1023,0.021503999829292297
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,1023,0.04351999859015147
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,2047,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,2047,0.06551466882228851
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,4095,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,4095,0.10852799812952678
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,8191,0.01621333385507266
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,8191,0.1957333286603292
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,16383,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,16383,0.3696639935175578
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,32767,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,32767,0.7195466359456381
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,65535,0.036864000062147774
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,1,0.007290666922926903
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,1,0.010890666395425797
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,65535,1.4305280049641926
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,3,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,7,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,15,0.0074560002734263735
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,15,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,31,0.007360000163316727
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,31,0.011946666985750198
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,63,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,63,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,255,0.008559999987483025
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,255,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,511,0.009754666437705358
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,511,0.03310933212439219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,1023,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,2047,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,1023,0.04351999859015147
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,2047,0.06553600231806438
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,4095,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,4095,0.10872000455856323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,8191,0.048469334840774536
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,8191,0.19575466712315878
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,16383,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,16383,0.36986132462819415
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,32767,0.03498666733503342
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,32767,0.7200160026550293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,65535,0.05495466788609823
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,65535,1.4312106768290203
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,1,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,7,0.007466666400432587
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,15,0.0074506668994824094
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,15,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,31,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,31,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,63,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,63,0.00816000004609426
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,127,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,127,0.009701333319147428
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,255,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,255,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,511,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,511,0.03379199902216593
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,1023,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,1023,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,2047,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,2047,0.06506666541099548
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,4095,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,4095,0.10871466994285583
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,8191,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,8191,0.1960960030555725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,16383,0.03583999971548716
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,16383,0.3705173333485921
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,32767,0.05478399991989136
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,32767,0.7191893259684244
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,65535,0.20838399728139242
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,1,0.00730666642387708
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,1,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,3,0.007471999774376552
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,65535,1.4315679868062336
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,3,0.010949333508809408
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,7,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,7,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,15,0.007631999750932057
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,15,0.010944000134865442
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,31,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,31,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,63,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,127,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,63,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,127,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,255,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,255,0.01381333296497663
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,511,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,511,0.0314026673634847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,1023,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,1023,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,2047,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,2047,0.0652213344971339
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,4095,0.025258667767047882
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,4095,0.1088853379090627
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,8191,0.03703466554482778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,8191,0.1959786613782247
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,16383,0.05853866537412008
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,16383,0.3712000052134196
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,32767,0.20309333006540933
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,32767,0.7200427055358887
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,65535,0.3959466616312663
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,1,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,65535,1.4602239926656086
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,3,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,3,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,7,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,7,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,15,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,31,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,31,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,63,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,63,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,127,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,127,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,255,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,255,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,511,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,511,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,1023,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,1023,0.034645333886146545
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,2047,0.04898133377234141
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,2047,0.025941332181294758
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,1,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,4095,0.06656000018119812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,1,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,4095,0.04232533276081085
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,3,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,3,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,7,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,7,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,15,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,15,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,31,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,63,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,31,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,63,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,127,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,127,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,255,0.018944000204404194
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,255,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,511,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,1023,0.03908266623814901
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,511,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,1023,0.02628266563018163
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,2047,0.05751466751098633
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,2047,0.04437333345413208
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,4095,0.20565332969029745
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,1,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,4095,0.0769706666469574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,3,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,1,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,3,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,7,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,7,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,15,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,15,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,31,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,31,0.0170666662355264
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,63,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,63,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,127,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,127,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,255,0.030720000465710957
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,255,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,511,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,511,0.032085334261258446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,1023,0.05751466751098633
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,1023,0.04744533201058706
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,2047,0.20599466562271118
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,2047,0.08089600006739299
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,1,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,1,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,4095,0.3998719851175944
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,4095,0.19694934288660684
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,3,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,3,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,7,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,7,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,15,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,15,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,31,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,31,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,63,0.028330666323502857
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,63,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,127,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,127,0.03601066768169403
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,255,0.03703466554482778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,255,0.043178667624791466
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,511,0.053247998158137
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,511,0.056661332647005715
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,1023,0.20241065820058188
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,1023,0.08703999718030293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,2047,0.3993599812189738
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,2047,0.20070399840672812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,1,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,1,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,4095,0.3887786865234375
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,4095,0.7879679997762045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,3,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,3,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,7,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,7,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,15,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,15,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,31,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,31,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,63,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,63,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,127,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,255,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,255,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,511,0.03583999971548716
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,511,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,1023,0.0481279989083608
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,1023,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,1,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,1,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,3,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,3,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,7,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,7,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,15,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,15,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,31,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,31,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,63,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,63,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,127,0.016895999511082966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,127,0.021503999829292297
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,255,0.031231999397277832
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,255,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,511,0.04113066693147024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,511,0.03310933212439219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,1023,0.0576853354771932
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,1,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,1,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,1023,0.048298666874567665
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,3,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,3,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,7,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,7,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,15,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,15,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,31,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,31,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,63,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,127,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,63,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,127,0.03703466554482778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,255,0.04334933559099833
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,255,0.0363520011305809
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,511,0.05256533126036326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,511,0.05853866537412008
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,1023,0.20480000972747803
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,1,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,1023,0.08806399504343669
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,1,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,3,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,3,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,7,0.05034666756788889
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,7,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,15,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,15,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,31,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,31,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,63,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,63,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,127,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,127,0.06468266745408376
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,255,0.06604800124963124
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,255,0.07918933530648549
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,511,0.20189867417017618
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,511,0.10393599669138591
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,1023,0.3952639897664388
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,1,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,1023,0.1967786749204
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,1,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,3,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,3,0.022357332209746044
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,7,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,7,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,15,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,15,0.022015998760859173
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,31,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,31,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,63,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,127,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,63,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,127,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,255,0.03822933385769526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,255,0.02679466704527537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,511,0.0506879985332489
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,511,0.03703466554482778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,1,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,1,0.03089066594839096
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,3,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,3,0.03089066594839096
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,7,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,7,0.03089066594839096
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,15,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,15,0.03089066594839096
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,31,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,31,0.03089066594839096
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,63,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,127,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,127,0.038058665891488395
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,63,0.03089066594839096
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,255,0.038058665891488395
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,255,0.045909335215886436
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,511,0.05461333195368449
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,511,0.06178133189678192
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,1,0.051541333397229515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,1,0.05239466826121012
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,3,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,3,0.05239466826121012
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,7,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,7,0.05239466826121012
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,15,0.051541333397229515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,15,0.05239466826121012
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,31,0.05239466826121012
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,31,0.051541333397229515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,63,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,127,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,63,0.05239466826121012
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,127,0.06656000018119812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,255,0.06690133114655812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,255,0.08004266520341237
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,1,0.09454933802286784
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,511,0.20684800545374551
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,511,0.10854400197664897
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,1,0.09642666578292847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,3,0.09642666578292847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,3,0.09454933802286784
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,7,0.09454933802286784
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,7,0.09642666578292847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,15,0.09437867005666097
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,15,0.09659733374913533
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,31,0.09454933802286784
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,31,0.09642666578292847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,63,0.09454933802286784
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,63,0.09642666578292847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,127,0.09523199995358785
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,127,0.12322133779525757
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,255,0.2034346659978231
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,255,0.1508693297704061
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,1,0.006826666494210561
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,511,0.39628799756368
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,511,0.21538132429122925
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,1,0.018922666708628338
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,3,0.006826666494210561
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,3,0.018944000204404194
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,7,0.007104000076651573
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,7,0.01911466692884763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,15,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,31,0.007658666620651881
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,15,0.018944000204404194
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,31,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,63,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,63,0.024405332903067272
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,127,0.008240000034372011
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,127,0.024405332903067272
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,255,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,255,0.025610665480295818
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,511,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,1023,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,511,0.030378667016824085
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,1023,0.051541333397229515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,2047,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,2047,0.09454933802286784
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,4095,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,4095,0.17923200130462646
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,8191,0.020992000897725422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,8191,0.35174401601155597
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,16383,0.695466677347819
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,16383,0.025941332181294758
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,32767,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,32767,1.4337706565856934
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,1,0.006826666494210561
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,65535,0.06144000093142191
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,65535,2.8528639475504556
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,1,0.019626667102177937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,3,0.006655999769767125
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,7,0.007157333195209503
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,3,0.019285333653291065
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,7,0.01911466692884763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,15,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,15,0.019626667102177937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,31,0.020842666427294414
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,31,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,63,0.024405332903067272
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,63,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,127,0.024746666351954143
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,127,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,255,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,255,0.025770666698614757
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,511,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,511,0.030154667794704437
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,1023,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,1023,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,2047,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,2047,0.09437867005666097
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,4095,0.1795413295427958
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,4095,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,8191,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,8191,0.35123201211293537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,16383,0.03566933423280716
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,16383,0.6959786415100098
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,32767,0.055125330885251365
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,65535,0.2070186734199524
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,32767,1.432693322499593
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,1,0.01911466692884763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,1,0.006826666494210561
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,65535,2.854058583577474
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,3,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,3,0.019434666881958645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,7,0.019285333653291065
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,15,0.0194560003777345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,31,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,15,0.007151999821265538
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,63,0.008229333286484083
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,31,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,63,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,127,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,127,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,255,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,255,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,511,0.030720000465710957
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,511,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,1023,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,1023,0.051541333397229515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,2047,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,2047,0.09437867005666097
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,4095,0.025258667767047882
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,4095,0.18040533860524496
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,8191,0.03703466554482778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,16383,0.05853866537412008
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,8191,0.35140268007914227
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,16383,0.696832021077474
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,32767,0.2053119937578837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,1,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,32767,1.4339359601338704
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,65535,0.3968000014623006
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,1,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,3,0.007344000041484833
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,3,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,7,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,65535,2.9044052759806314
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,7,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,15,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,31,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,63,0.008527999743819237
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,63,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,31,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,127,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,255,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,511,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,511,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,1023,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,2047,0.02679466704527537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,1023,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,2047,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,4095,0.040448000033696495
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,4095,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,8191,0.05751466751098633
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,8191,0.07338666419188182
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,16383,0.20206934213638306
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,16383,0.13858133554458618
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,32767,0.39611732959747314
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,32767,0.27187200387318927
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,65535,0.7808000246683756
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,65535,0.5338453451792399
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,float16,1,0.040106666584809623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,fp8,1,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,float16,3,0.040106666584809623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,fp8,3,0.040789333482583366
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,float16,7,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,fp8,7,0.04095999896526337
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,fp8,15,0.040618665516376495
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,float16,15,0.040106666584809623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,fp8,31,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,float16,31,0.040106666584809623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,float16,63,0.040106666584809623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,fp8,63,0.04113066693147024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,float16,127,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,fp8,127,0.04215466479460398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,float16,1,0.05358933409055074
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,float16,255,0.048298666874567665
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,fp8,1,0.06348800162474315
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,fp8,255,0.04949333270390829
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,float16,3,0.05358933409055074
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,fp8,3,0.05461333195368449
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,fp8,7,0.05444266895453135
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,float16,7,0.05358933409055074
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,float16,15,0.05358933409055074
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,fp8,15,0.05444266895453135
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,fp8,31,0.05478399991989136
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,float16,31,0.05341866612434387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,float16,63,0.05358933409055074
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,fp8,63,0.05461333195368449
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,float16,127,0.05358933409055074
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,fp8,127,0.06894933183987935
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,float16,255,0.06894933183987935
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,float16,1,0.09659733374913533
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,fp8,1,0.09864532947540283
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,fp8,255,0.0846506655216217
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,float16,3,0.09659733374913533
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,fp8,3,0.09864532947540283
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,float16,7,0.09659733374913533
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,fp8,7,0.09864532947540283
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,float16,15,0.09659733374913533
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,fp8,15,0.09864532947540283
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,float16,31,0.09659733374913533
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,fp8,31,0.09847467144330342
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,float16,63,0.09659733374913533
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,fp8,63,0.09864532947540283
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,float16,127,0.09745066364606221
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,fp8,127,0.1262933313846588
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,fp8,1,0.18756266434987387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,float16,255,0.21401600042978922
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,float16,1,0.18568533658981323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,fp8,3,0.18773333231608072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,float16,3,0.1858560045560201
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,fp8,255,0.15359999736150107
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,float16,7,0.18551466862360635
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,fp8,7,0.18773333231608072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,float16,15,0.18551466862360635
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,fp8,15,0.18773333231608072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,fp8,31,0.18773333231608072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,float16,31,0.18568533658981323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,float16,63,0.18568533658981323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,fp8,63,0.18773333231608072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,float16,127,0.25309866666793823
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,fp8,127,0.24012800057729086
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,fp8,255,0.31624533732732135
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,float16,255,0.43263999621073407
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,float16,1,0.07287466526031494
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,fp8,1,0.07389866809050243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,fp8,3,0.07389866809050243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,float16,3,0.0730453332265218
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,float16,7,0.0727040022611618
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,fp8,7,0.07389866809050243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,float16,15,0.0730453332265218
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,fp8,15,0.07389866809050243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,float16,31,0.07287466526031494
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,fp8,31,0.07355733215808868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,float16,63,0.07287466526031494
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,fp8,63,0.07389866809050243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,float16,127,0.0730453332265218
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,float16,1,0.10171733299891154
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,fp8,1,0.10342400272687276
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,fp8,127,0.07714133461316426
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,float16,3,0.10154666503270467
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,float16,7,0.10120532910029094
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,fp8,3,0.10257066289583842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,fp8,7,0.10376532872517903
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,float16,15,0.10154666503270467
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,fp8,15,0.10342400272687276
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,float16,31,0.10120532910029094
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,fp8,31,0.1032533347606659
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,float16,63,0.10154666503270467
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,fp8,63,0.10291199882825215
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,float16,127,0.10171733299891154
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,fp8,127,0.13141333063443503
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,float16,1,0.19063466787338257
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,fp8,1,0.19268266359965006
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,float16,3,0.1919999917348226
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,fp8,3,0.19268266359965006
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,float16,7,0.19080533583958945
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,fp8,7,0.19268266359965006
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,float16,15,0.19080533583958945
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,fp8,15,0.19268266359965006
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,float16,31,0.19063466787338257
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,fp8,31,0.19234132766723633
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,float16,63,0.19063466787338257
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,fp8,63,0.19234132766723633
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,float16,127,0.2653866608937581
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,fp8,127,0.24627200762430826
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,float16,1,0.3689813216527303
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,fp8,1,0.3700053294499715
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,float16,3,0.36881065368652344
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,fp8,3,0.3700053294499715
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,float16,7,0.36881065368652344
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,fp8,7,0.3700053294499715
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,float16,15,0.3689813216527303
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,fp8,15,0.36983466148376465
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,float16,31,0.36983466148376465
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,float16,63,0.4140373468399048
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,fp8,31,0.36983466148376465
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,1,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,fp8,63,0.3700053294499715
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,1,0.018944000204404194
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,3,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,3,0.01911466692884763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,7,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,float16,127,0.497322678565979
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,fp8,127,0.5420373280843099
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,7,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,15,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,15,0.020138667275508244
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,31,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,31,0.023039999107519787
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,63,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,63,0.022831998765468597
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,127,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,127,0.023552000522613525
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,255,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,255,0.028501334289709728
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,511,0.05000533163547516
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,511,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,1023,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,1023,0.09290132919947307
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,2047,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,2047,0.18090667327245077
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,4095,0.0194560003777345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,4095,0.3556693394978841
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,8191,0.02867199977238973
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,8191,0.7154346307118734
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,16383,0.04113066693147024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,16383,2.133024056752523
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,1,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,32767,0.06382933259010315
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,1,0.019093333433071773
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,3,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,3,0.01912533367673556
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,32767,4.41429328918457
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,7,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,7,0.019797333826621372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,15,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,31,0.010960000256697336
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,15,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,31,0.023039999107519787
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,63,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,63,0.023210667073726654
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,127,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,127,0.023039999107519787
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,255,0.028501334289709728
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,511,0.05000533163547516
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,511,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,1023,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,1023,0.09335466225941975
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,2047,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,2047,0.18068265914916992
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,4095,0.025770666698614757
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,4095,0.35602132479349774
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,8191,0.03737599899371465
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,8191,0.7157333691914877
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,16383,0.05870933334032694
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,32767,0.20565332969029745
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,16383,2.135039965311686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,1,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,32767,4.422597249348958
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,1,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,3,0.00749333327015241
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,3,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,15,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,31,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,31,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,63,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,63,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,127,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,255,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,511,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,511,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,1023,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,1023,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,2047,0.02679466704527537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,2047,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,4095,0.04095999896526337
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,4095,0.04095999896526337
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,8191,0.0576853354771932
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,8191,0.0730453332265218
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,16383,0.20411733786265054
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,1,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,16383,0.13875200351079306
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,32767,0.39714133739471436
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,32767,0.2723840077718099
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,3,0.007471999774376552
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,7,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,7,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,15,0.007477333148320516
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,31,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,31,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,63,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,127,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,255,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,511,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,511,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,1023,0.028160000840822857
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,1023,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,2047,0.039936001102129616
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,2047,0.024746666351954143
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,4095,0.056661332647005715
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,4095,0.04095999896526337
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,8191,0.07372800012429555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,8191,0.20241065820058188
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,16383,0.39953064918518066
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,16383,0.19643733898798624
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,32767,0.7814826965332031
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,1,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,32767,0.37410132090250653
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,1,0.029525332152843475
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,3,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,3,0.03054933249950409
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,7,0.010559999694426855
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,7,0.031066666046778362
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,15,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,15,0.03925866633653641
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,31,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,31,0.039093332986036934
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,63,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,63,0.03990400085846583
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,127,0.0170666662355264
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,127,0.049312000473340355
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,255,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,255,0.08942932883898418
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,511,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,511,0.17083734273910522
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,1023,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,1023,0.33433600266774494
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,2047,0.027647999425729115
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,2047,0.6598560015360514
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,4095,0.040618665516376495
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,8191,0.05973333120346069
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,4095,1.3122560183207195
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,8191,2.822655995686849
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,16383,0.23586134115854898
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,1,0.0084906667470932
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,3,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,16383,8.452437082926432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,7,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,15,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,31,0.008352000266313553
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,63,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,255,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,255,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,511,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,511,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,1023,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,1023,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,2047,0.03515733281771342
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,2047,0.04215466479460398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,4095,0.05751466751098633
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,4095,0.07492266595363617
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,8191,0.22033067544301352
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,8191,0.14045866330464682
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,16383,0.42871467272440594
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,16383,0.2764799992243449
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,3,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,7,0.008352000266313553
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,15,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,31,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,15,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,63,0.008522666369875273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,63,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,255,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,255,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,511,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,1023,0.03601066768169403
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,511,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,1023,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,2047,0.05563733478387197
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,2047,0.04215466479460398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,4095,0.21913599967956543
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,4095,0.07509333391984303
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,8191,0.4271786610285441
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,8191,0.21623466412226358
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,16383,0.8424106438954672
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,16383,0.41198933124542236
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,1,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,1,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,3,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,3,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,7,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,7,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,15,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,15,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,63,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,63,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,127,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,127,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,255,0.017407999684413273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,255,0.02218666672706604
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,511,0.03618133316437403
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,511,0.02679466704527537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,31,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,31,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,1023,0.05205333232879639
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,1023,0.04334933559099833
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,4095,0.4247893492380778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,4095,0.2193066676457723
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,8191,0.8381439844767252
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,8191,0.4145493507385254
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,16383,0.8103253046671549
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,16383,1.6701439221700032
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,1,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,3,0.011978667229413986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,2047,0.21606399615605673
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,7,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,2047,0.07816533247629802
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,7,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,15,0.007818666597207388
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,15,0.012810666114091873
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,31,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,31,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,63,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,63,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,127,0.014666666587193808
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,127,0.008549333239595095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,255,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,255,0.0194560003777345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,511,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,511,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,1023,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,1023,0.03618133316437403
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,2047,0.02218666672706604
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,2047,0.05853866537412008
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,4095,0.03398933261632919
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,4095,0.1013759970664978
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,8191,0.018944000204404194
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,8191,0.15974400440851846
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,16383,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,16383,0.2984960079193115
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,32767,0.025941332181294758
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,32767,0.5855199893315634
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,65535,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,1,0.007285333548982938
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,65535,1.1518293221791585
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,1,0.011946666985750198
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,3,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,7,0.012432000289360682
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,15,0.00766933336853981
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,15,0.012810666114091873
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,31,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,31,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,63,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,127,0.009370666618148485
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,63,0.014671999961137772
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,127,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,255,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,255,0.019658666104078293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,511,0.011946666985750198
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,511,0.020138667275508244
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,1023,0.012991999586423239
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,1023,0.03566933423280716
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,2047,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,2047,0.0554666668176651
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,4095,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,4095,0.09318400422732036
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,8191,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,8191,0.16059733430544534
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,16383,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,16383,0.2986666758855184
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,32767,0.021162666380405426
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,65535,0.028501334289709728
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,32767,0.5857066710789999
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,1,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,1,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,65535,1.1518293221791585
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,3,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,3,0.012666666259368261
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,7,0.007327999919652939
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,7,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,15,0.0074986666440963745
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,15,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,31,0.007877333089709282
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,31,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,63,0.014565333724021912
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,127,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,255,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,255,0.0194560003777345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,511,0.011946666985750198
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,511,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,1023,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,1023,0.03532800078392029
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,2047,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,2047,0.05596266686916351
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,4095,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,4095,0.0942080020904541
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,8191,0.01621333385507266
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,8191,0.16059733430544534
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,16383,0.018432000031073887
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,16383,0.2995199958483378
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,32767,0.032255999743938446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,32767,0.5863893429438273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,1,0.0084906667470932
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,1,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,65535,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,65535,1.1528533299763997
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,7,0.007322666545708974
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,15,0.007231999809543292
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,15,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,31,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,31,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,127,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,63,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,127,0.015210667004187902
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,3,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,255,0.019968000551064808
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,7,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,511,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,511,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,1023,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,2047,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,1023,0.03395200024048487
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,2047,0.05547733108202616
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,4095,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,4095,0.09337066610654195
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,8191,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,255,0.008517333616813024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,8191,0.16127999623616537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,16383,0.02679466704527537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,16383,0.30003199974695843
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,32767,0.04351999859015147
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,32767,0.5867520173390707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,65535,0.22818134228388467
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,1,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,3,0.007322666545708974
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,65535,1.1530240376790364
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,3,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,7,0.007135999699433644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,7,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,15,0.029525332152843475
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,31,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,31,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,63,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,127,0.014874666929244995
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,255,0.0194560003777345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,255,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,511,0.035802667339642845
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,511,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,1023,0.022357332209746044
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,1023,0.054586668809254967
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,2047,0.0315733328461647
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,2047,0.08345599969228108
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,4095,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,4095,0.13926399747530618
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,8191,0.019797333826621372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,8191,0.25497599442799884
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,16383,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,16383,0.4920320113499959
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,32767,0.026965332527955372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,32767,0.9603412946065267
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,1,0.007327999919652939
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,1,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,65535,0.034474665919939675
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,65535,1.4349652926127117
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,3,0.012671999633312225
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,7,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,7,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,15,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,31,0.008165333420038223
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,31,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,127,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,127,0.015168000012636185
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,255,0.020154666155576706
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,511,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,511,0.0339626669883728
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,1023,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,1023,0.050474668542544045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,2047,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,2047,0.07765333354473114
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,4095,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,8191,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,4095,0.1389226714769999
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,8191,0.25514666239420575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,16383,0.018432000031073887
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,16383,0.4922026793162028
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,32767,0.032255999743938446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,32767,0.9608533382415771
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,65535,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,1,0.012421333541472753
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,3,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,65535,1.4346240361531575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,3,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,7,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,15,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,31,0.007871999715765318
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,31,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,127,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,63,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,127,0.015509333461523056
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,255,0.019626667102177937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,511,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,511,0.036362667878468834
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,1023,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,1023,0.0506879985332489
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,2047,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,2047,0.07833600044250488
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,4095,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,4095,0.139082670211792
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,8191,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,8191,0.25566933552424115
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,16383,0.027818667391935985
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,16383,0.49307199319203693
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,32767,0.04488533238569895
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,32767,0.9613760312398275
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,65535,0.23142399390538534
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,1,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,65535,1.4354772567749023
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,3,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,7,0.007482666522264481
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,7,0.012954667210578918
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,15,0.007477333148320516
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,15,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,31,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,31,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,255,0.019968000551064808
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,255,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,511,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,511,0.0339626669883728
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,1023,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,1023,0.0506879985332489
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,2047,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,2047,0.07816533247629802
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,4095,0.019797333826621372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,4095,0.13924266894658408
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,8191,0.030720000465710957
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,8191,0.25594667593638104
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,16383,0.04454400142033895
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,16383,0.493397315343221
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,32767,0.22272000710169473
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,32767,0.9618773460388184
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,1,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,65535,0.4391253391901652
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,1,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,65535,1.4590293566385906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,3,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,3,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,7,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,7,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,15,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,15,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,31,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,63,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,63,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,127,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,127,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,255,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,255,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,511,0.01791999985774358
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,511,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,1023,0.02679466704527537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,1023,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,2047,0.04232533276081085
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,2047,0.04334933559099833
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,4095,0.06690133114655812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,4095,0.07611733178297679
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,8191,0.23347200949986777
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,1,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,1,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,8191,0.14199466506640115
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,3,0.008522666369875273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,7,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,7,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,15,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,31,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,31,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,63,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,127,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,127,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,255,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,255,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,511,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,511,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,1023,0.03703466554482778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,1023,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,2047,0.05614933371543884
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,2047,0.04232533276081085
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,4095,0.22152533133824667
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,4095,0.07509333391984303
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,8191,0.42905600865681964
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,8191,0.21708800395329794
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,1,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,1,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,3,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,3,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,7,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,7,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,15,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,15,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,31,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,31,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,63,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,63,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,127,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,127,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,255,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,255,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,511,0.03703466554482778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,511,0.02679466704527537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,1023,0.053247998158137
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,1023,0.04334933559099833
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,2047,0.21913599967956543
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,2047,0.07816533247629802
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,4095,0.42444801330566406
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,4095,0.2187946637471517
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,1,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,1,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,8191,0.8360959688822428
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,8191,0.4179626703262329
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,3,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,3,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,7,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,7,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,15,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,15,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,63,0.016895999511082966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,63,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,31,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,31,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,127,0.021162666380405426
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,127,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,255,0.036864000062147774
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,255,0.029696000119050343
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,511,0.05341866612434387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,511,0.04949333270390829
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,1023,0.21435733636220297
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,1023,0.08226133386294048
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,2047,0.42342400550842285
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,2047,0.22169599930445352
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,4095,0.8360959688822428
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,4095,0.4150613149007161
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,1,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,8191,0.802133321762085
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,8191,1.6674133936564128
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,3,0.00730666642387708
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,3,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,7,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,7,0.012485332787036896
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,15,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,31,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,63,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,63,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,127,0.008656000097592672
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,255,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,255,0.03324799984693527
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,511,0.02181333303451538
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,511,0.045909335215886436
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,1023,0.026629333694775898
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,1023,0.06709333260854085
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,2047,0.016042667130629223
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,2047,0.1114453375339508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,4095,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,8191,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,4095,0.2001919945081075
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,8191,0.37666134039560956
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,16383,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,16383,0.7290879885355631
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,32767,0.04113066693147024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,1,0.012309333930412928
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,32767,1.4346240361531575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,65535,0.053930665055910744
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,65535,2.865989367167155
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,3,0.007322666545708974
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,3,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,7,0.007349333415428798
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,7,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,15,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,31,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,31,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,63,0.008693333094318708
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,63,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,127,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,127,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,255,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,255,0.03278400003910065
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,511,0.022015998760859173
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,511,0.04553600152333578
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,1023,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,2047,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,1023,0.067071999112765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,2047,0.11076266566912334
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,4095,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,4095,0.20036266247431436
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,8191,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,8191,0.3778560161590576
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,16383,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,16383,0.7297706604003906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,32767,0.04539733131726583
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,32767,1.4349652926127117
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,65535,0.2310826579729716
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,1,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,65535,2.862144152323405
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,3,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,7,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,15,0.00766933336853981
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,15,0.013002666334311167
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,31,0.01421333352724711
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,63,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,63,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,127,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,127,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,255,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,255,0.03345066557327906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,511,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,511,0.045514668027559914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,1023,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,1023,0.06729599833488464
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,2047,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,2047,0.1129813293615977
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,4095,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,4095,0.20137600104014078
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,8191,0.032085334261258446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,8191,0.37751468022664386
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,16383,0.045738667249679565
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,16383,0.7307946681976318
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,32767,0.22630399465560913
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,65535,0.4427093267440796
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,32767,1.4363306363423665
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,1,0.012576000144084295
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,3,0.0075040000180403394
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,65535,2.9034080505371094
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,3,0.012762666990359625
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,15,0.007520000139872233
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,7,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,15,0.01314666618903478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,31,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,31,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,63,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,63,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,127,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,127,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,255,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,511,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,255,0.03328000009059906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,511,0.045738667249679565
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,1023,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,1023,0.06741333504517873
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,2047,0.022015998760859173
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,2047,0.11161599556605022
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,4095,0.03310933212439219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,4095,0.20155733823776245
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,8191,0.049322664737701416
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,8191,0.4010666608810425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,16383,0.2208426594734192
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,16383,0.7313066323598226
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,32767,0.43161598841349286
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,32767,1.4581759770711262
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,65535,0.8596479892730713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,1,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,3,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,1,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,3,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,65535,2.9071359634399414
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,7,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,7,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,15,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,15,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,31,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,31,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,63,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,63,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,127,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,255,0.018090666582187016
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,127,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,255,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,511,0.027306665976842243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,511,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,1023,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,1023,0.0264533335963885
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,2047,0.06604800124963124
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,2047,0.04539733131726583
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,1,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,4095,0.23466666539510092
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,1,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,3,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,4095,0.07765333354473114
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,3,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,7,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,7,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,15,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,31,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,15,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,31,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,63,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,63,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,255,0.023210667073726654
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,255,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,511,0.03788800040880839
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,511,0.027647999425729115
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,1023,0.053930665055910744
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,1023,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,2047,0.2244266668955485
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,2047,0.07816533247629802
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,4095,0.43298133214314777
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,1,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,1,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,3,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,4095,0.22272000710169473
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,3,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,7,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,7,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,15,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,15,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,31,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,31,0.020992000897725422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,63,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,63,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,127,0.020992000897725422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,127,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,255,0.03737599899371465
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,255,0.029696000119050343
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,511,0.05461333195368449
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,511,0.04949333270390829
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,1023,0.2198186715443929
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,1023,0.08226133386294048
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,2047,0.43059198061625165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,2047,0.22408533096313477
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,1,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,4095,0.41557331879933673
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,4095,0.8434346516927084
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,1,0.03703466554482778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,3,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,3,0.037205333511034645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,7,0.03703466554482778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,7,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,15,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,15,0.03703466554482778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,31,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,31,0.037205333511034645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,63,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,63,0.037205333511034645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,127,0.037205333511034645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,127,0.04215466479460398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,255,0.04915200173854828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,255,0.05358933409055074
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,511,0.21708800395329794
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,511,0.09318400422732036
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,1023,0.43110398451487225
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,1023,0.2244266668955485
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,2047,0.8495786984761556
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,2047,0.42820266882578534
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,1,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,1,0.017237332959969837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,4095,0.8345599969228109
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,3,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,4095,1.6880639394124348
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,3,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,7,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,7,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,15,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,31,0.017407999684413273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,15,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,31,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,63,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,127,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,127,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,255,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,255,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,511,0.03242666771014532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,511,0.04539733131726583
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,1023,0.06468266745408376
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,1023,0.048810665806134544
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,1,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,3,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,1,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,3,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,7,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,7,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,15,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,15,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,31,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,31,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,63,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,63,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,127,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,127,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,255,0.03925333420435587
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,255,0.03054933249950409
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,511,0.0554666668176651
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,511,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,1023,0.22613332668940225
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,1023,0.08328533172607422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,1,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,1,0.03822933385769526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,3,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,3,0.03822933385769526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,7,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,7,0.03822933385769526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,15,0.038058665891488395
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,15,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,31,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,31,0.03822933385769526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,63,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,63,0.038058665891488395
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,127,0.03822933385769526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,127,0.043178667624791466
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,255,0.05222400029500326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,255,0.054272000988324486
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,511,0.2228906750679016
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,511,0.09318400422732036
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,1023,0.4396373430887858
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,1023,0.23040000597635904
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,1,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,1,0.06690133114655812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,3,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,3,0.06690133114655812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,7,0.051541333397229515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,7,0.06690133114655812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,15,0.051541333397229515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,15,0.06690133114655812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,31,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,31,0.06690133114655812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,63,0.051541333397229515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,127,0.06690133114655812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,63,0.06690133114655812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,127,0.0769706666469574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,255,0.2152106761932373
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,255,0.09830400347709656
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,511,0.42769066492716473
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,511,0.2172586719195048
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,1023,0.8461653391520182
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,1023,0.4208639860153198
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,1,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,1,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,3,0.024746666351954143
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,3,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,7,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,7,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,15,0.024405332903067272
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,15,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,31,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,31,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,63,0.024746666351954143
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,63,0.030207999050617218
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,127,0.028330666323502857
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,127,0.03310933212439219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,255,0.04761599997679392
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,255,0.03976533313592275
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,1,0.032085334261258446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,1,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,511,0.05922133227189382
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,511,0.0679253339767456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,3,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,3,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,7,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,7,0.040106666584809623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,15,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,15,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,31,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,31,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,63,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,63,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,127,0.040106666584809623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,127,0.04437333345413208
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,255,0.05273599922657013
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,255,0.056832000613212585
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,511,0.23278933763504028
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,1,0.05341866612434387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,511,0.09471999605496724
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,1,0.06894933183987935
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,3,0.05341866612434387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,3,0.06894933183987935
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,7,0.05358933409055074
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,7,0.06877866884072621
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,15,0.05358933409055074
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,15,0.06877866884072621
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,31,0.06894933183987935
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,31,0.05358933409055074
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,63,0.05341866612434387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,63,0.06877866884072621
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,127,0.06894933183987935
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,127,0.07816533247629802
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,255,0.22476800282796225
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,255,0.09966933727264404
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,511,0.43878400325775146
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,511,0.22698666652043661
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,1,0.09642666578292847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,1,0.1262933313846588
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,3,0.09659733374913533
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,3,0.1262933313846588
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,7,0.12612266341845194
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,7,0.09659733374913533
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,15,0.09659733374913533
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,15,0.1262933313846588
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,31,0.09642666578292847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,31,0.1262933313846588
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,63,0.09762133161226909
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,63,0.1262933313846588
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,127,0.20974934101104736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,127,0.14574933052062988
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,255,0.4247893492380778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,255,0.22152533133824667
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,511,0.8429226875305176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,1,0.006826666494210561
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,1,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,511,0.4352000157038371
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,3,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,3,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,7,0.02081599955757459
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,15,0.007717333113153775
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,15,0.022015998760859173
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,31,0.008176000167926153
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,31,0.025258667767047882
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,63,0.008170666793982187
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,63,0.025429333249727886
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,127,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,255,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,127,0.025600001215934753
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,255,0.0315733328461647
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,511,0.017237332959969837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,511,0.05256533126036326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,1023,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,1023,0.09540266791979472
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,2047,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,2047,0.1815413236618042
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,4095,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,4095,0.35398932298024494
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,8191,0.028160000840822857
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,8191,0.6983839670817057
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,16383,0.03515733281771342
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,16383,1.3853012720743816
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,32767,0.05580799778302511
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,32767,2.8631038665771484
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,1,0.006826666494210561
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,65535,0.2392746607462565
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,3,0.007151999821265538
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,1,0.021141332884629566
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,3,0.021333334346612293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,7,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,65535,5.714944203694661
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,7,0.021333334346612293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,15,0.02250666668017705
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,31,0.02555199960867564
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,31,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,63,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,63,0.025600001215934753
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,127,0.008879999940594038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,127,0.025770666698614757
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,255,0.011274666835864386
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,255,0.0314026673634847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,511,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,511,0.05256533126036326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,1023,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,1023,0.09557333588600159
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,2047,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,2047,0.18158932526906332
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,4095,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,4095,0.35415999094645184
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,8191,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,16383,0.045738667249679565
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,16383,1.3880267143249512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,8191,0.6987360318501791
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,32767,0.22510933876037598
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,1,0.006826666494210561
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,65535,0.4391253391901652
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,32767,2.86356258392334
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,1,0.021333334346612293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,3,0.021317332983016968
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,65535,5.781375885009766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,7,0.021312000850836437
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,15,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,15,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,31,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,31,0.025600001215934753
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,63,0.025770666698614757
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,63,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,127,0.009343999748428663
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,127,0.025770666698614757
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,255,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,255,0.03143466760714849
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,511,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,511,0.052906667192777
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,1023,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,1023,0.09623466928799947
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,2047,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,2047,0.18175999323527017
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,4095,0.03310933212439219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,8191,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,4095,0.35447466373443604
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,8191,0.7004160086313883
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,16383,0.2228906750679016
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,16383,1.3864960670471191
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,32767,0.4341760079065959
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,32767,2.9018452962239585
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,65535,0.8608427047729492
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,65535,5.792250951131185
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,3,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,7,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,31,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,31,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,63,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,127,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,127,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,255,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,255,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,511,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,511,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,1023,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,1023,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,2047,0.034304000437259674
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,2047,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,4095,0.07406933108965556
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,4095,0.0554666668176651
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,8191,0.21572266022364298
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,8191,0.13960533340771994
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,16383,0.4246186812718709
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,16383,0.27562665939331055
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,32767,0.8506027062733968
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,32767,0.5369173288345337
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,65535,1.691989262898763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,1,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,65535,1.0615466435750325
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,fp8,1,0.052906667192777
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,3,0.04351999859015147
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,fp8,3,0.053247998158137
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,7,0.04369066655635834
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,fp8,7,0.05256533126036326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,15,0.04369066655635834
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,fp8,15,0.052906667192777
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,31,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,fp8,31,0.053077335158983864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,63,0.04437333345413208
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,fp8,63,0.05341866612434387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,127,0.05205333232879639
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,fp8,127,0.059903999169667564
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,255,0.062122667829195656
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,1,0.05734399954477946
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,fp8,255,0.07065600156784058
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,fp8,1,0.0718506673971812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,3,0.05734399954477946
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,fp8,3,0.07202133536338806
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,7,0.05734399954477946
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,15,0.05751466751098633
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,fp8,7,0.0721919983625412
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,31,0.05734399954477946
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,fp8,15,0.0721919983625412
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,fp8,31,0.0718506673971812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,63,0.05853866537412008
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,fp8,63,0.0721919983625412
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,127,0.07338666419188182
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,fp8,127,0.08106666803359985
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,fp8,1,0.12987732887268066
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,1,0.10069333513577779
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,fp8,255,0.10342400272687276
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,255,0.24371200799942017
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,3,0.1327786644299825
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,fp8,3,0.1295360028743744
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,7,0.10052266716957092
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,fp8,7,0.12970667084058127
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,15,0.10052266716957092
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,fp8,15,0.12970667084058127
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,31,0.10069333513577779
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,fp8,31,0.1295360028743744
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,63,0.10171733299891154
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,fp8,63,0.1293653349081675
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,127,0.2310826579729716
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,fp8,127,0.14899200201034546
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,1,0.18961066007614136
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,255,0.44817066192626953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,fp8,255,0.24183466037114462
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,fp8,1,0.24900267521540323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,3,0.18978132804234824
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,fp8,3,0.24849067131678262
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,7,0.18978132804234824
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,fp8,7,0.24900267521540323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,15,0.18978132804234824
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,fp8,15,0.2486613392829895
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,31,0.20155733823776245
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,fp8,31,0.24849067131678262
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,63,0.272213339805603
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,fp8,63,0.2491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,127,0.42632532119750977
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,fp8,127,0.3237546682357788
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,255,0.839680035909017
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,fp8,255,0.4391253391901652
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,float16,1,0.07816533247629802
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,fp8,1,0.09591466188430786
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,float16,3,0.07816533247629802
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,fp8,3,0.09574400385220845
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,float16,7,0.07833600044250488
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,fp8,7,0.09608532985051473
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,float16,15,0.07816533247629802
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,fp8,15,0.0962559978167216
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,fp8,31,0.09642666578292847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,float16,63,0.07867733140786488
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,float16,31,0.07850666840871175
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,fp8,63,0.09591466188430786
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,float16,127,0.17100799083709717
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,fp8,127,0.1109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,float16,1,0.10615467031796773
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,fp8,1,0.13499733805656433
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,fp8,3,0.1353386640548706
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,float16,3,0.1063253382841746
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,fp8,7,0.1353386640548706
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,float16,7,0.10666666428248088
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,float16,15,0.10666666428248088
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,fp8,15,0.1353386640548706
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,float16,31,0.1083733340104421
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,fp8,31,0.13516799608866373
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,float16,63,0.16810667514801025
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,fp8,63,0.13499733805656433
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,float16,127,0.2752853234608968
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,float16,1,0.1955839991569519
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,fp8,127,0.1914880077044169
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,fp8,1,0.25412267446517944
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,float16,3,0.1955839991569519
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,fp8,3,0.25412267446517944
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,float16,7,0.19575466712315878
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,fp8,7,0.25412267446517944
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,fp8,15,0.25412267446517944
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,float16,15,0.1971199909845988
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,float16,31,0.2727253238360087
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,fp8,31,0.25514666239420575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,float16,63,0.3058346708615621
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,fp8,63,0.28757333755493164
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,float16,127,0.46745598316192627
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,float16,1,0.37410132090250653
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,fp8,1,0.4916906754175822
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,fp8,127,0.3575466473897298
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,fp8,3,0.4916906754175822
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,float16,3,0.37410132090250653
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,float16,7,0.37563733259836835
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,fp8,7,0.4916906754175822
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,float16,15,0.4241066773732503
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,fp8,15,0.4939093192418416
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,float16,31,0.5031253496805826
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,fp8,31,0.532480001449585
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,float16,63,0.5451093514760336
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,fp8,63,0.5708800156911215
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,float16,127,0.8470186392466227
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,1,0.010399999717871347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,1,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,3,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,fp8,127,0.6545066833496094
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,3,0.021333334346612293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,7,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,7,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,15,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,31,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,31,0.023552000522613525
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,63,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,127,0.014149333039919535
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,63,0.023898666103680927
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,127,0.029525332152843475
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,255,0.01791999985774358
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,255,0.05085866649945577
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,511,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,511,0.0942080020904541
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,1023,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,1023,0.18227199713389078
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,2047,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,2047,0.3568640152613322
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,4095,0.025941332181294758
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,4095,0.706613302230835
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,8191,0.03822933385769526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,8191,1.4267733891805012
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,16383,0.056320001681645714
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,32767,0.23415466149648032
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,16383,4.237653414408366
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,1,0.010405333091815313
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,1,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,3,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,3,0.02139200021823247
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,7,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,32767,8.68130111694336
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,7,0.02218666672706604
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,15,0.011258666714032492
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,15,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,31,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,31,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,63,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,63,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,255,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,127,0.057002668579419456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,255,0.051029334465662636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,511,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,511,0.09471999605496724
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,1023,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,1023,0.18210132916768393
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,2047,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,2047,0.3568640152613322
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,4095,0.03345066557327906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,4095,0.7065760294596354
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,8191,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,8191,1.4283092816670735
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,16383,0.2249386707941691
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,16383,4.241237322489421
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,32767,0.43110398451487225
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,1,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,3,0.008352000266313553
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,3,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,7,0.008336000144481659
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,32767,9.01580810546875
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,15,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,63,0.008517333616813024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,255,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,255,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,511,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,511,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,1023,0.02218666672706604
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,1023,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,2047,0.03498666733503342
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,2047,0.04215466479460398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,4095,0.05717333157857259
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,4095,0.07406933108965556
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,8191,0.21794132391611734
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,8191,0.13960533340771994
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,16383,0.42769066492716473
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,16383,0.2752853234608968
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,32767,0.851797342300415
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,1,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,32767,0.5387946764628092
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,3,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,3,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,7,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,7,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,31,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,63,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,255,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,255,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,511,0.02218666672706604
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,511,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,1023,0.03601066768169403
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,1023,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,2047,0.05478399991989136
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,2047,0.04215466479460398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,4095,0.21640533208847046
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,4095,0.07509333391984303
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,8191,0.4227413336435954
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,8191,0.21401600042978922
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,16383,0.8407039642333984
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,16383,0.411135991414388
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,32767,1.6786773999532063
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,32767,0.8065706888834635
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,1,0.0100426667680343
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,1,0.01911466692884763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,3,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,3,0.01911466692884763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,7,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,7,0.019952000429232914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,15,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,15,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,31,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,31,0.022848000129063923
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,63,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,63,0.023552000522613525
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,127,0.013839999834696451
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,127,0.023552000522613525
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,255,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,255,0.028624000648657482
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,511,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,511,0.050186668833096824
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,1023,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,1023,0.09335466225941975
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,2047,0.018090666582187016
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,2047,0.18141865730285645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,4095,0.025429333249727886
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,4095,0.355840007464091
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,8191,0.037717332442601524
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,8191,0.7157759666442871
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,16383,0.05853866537412008
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,1,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,16383,2.133333365122477
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,32767,0.20497065782546997
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,32767,4.41429328918457
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,7,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,7,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,31,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,63,0.007322666545708974
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,127,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,127,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,255,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,511,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,511,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,1023,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,1023,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,2047,0.02679466704527537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,2047,0.024746666351954143
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,4095,0.04113066693147024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,8191,0.0580266664425532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,8191,0.07372800012429555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,4095,0.04113066693147024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,16383,0.2206719915072123
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,16383,0.13943466544151306
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,32767,0.3906559944152832
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,32767,0.2727253238360087
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,3,0.008176000167926153
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,1,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,7,0.00749333327015241
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,15,0.007461333026488622
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,15,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,31,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,63,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,127,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,255,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,255,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,511,0.01791999985774358
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,511,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,1023,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,1023,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,2047,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,2047,0.024746666351954143
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,4095,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,4095,0.05751466751098633
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,8191,0.20258132616678873
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,8191,0.07389866809050243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,16383,0.39321601390838623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,16383,0.19404800732930502
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,32767,0.3752959966659546
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,32767,0.7821653683980306
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,1,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,1,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,3,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,7,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,31,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,7,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,15,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,31,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,63,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,15,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,63,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,127,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,127,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,255,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,255,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,511,0.02918400118748347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,511,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,1023,0.03908266623814901
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,1023,0.025770666698614757
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,2047,0.056832000613212585
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,2047,0.04351999859015147
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,4095,0.20104533433914185
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,4095,0.07611733178297679
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,8191,0.19592533508936563
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,8191,0.39202133814493817
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,16383,0.7799466451009115
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,16383,0.3720533450444539
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,32767,1.5580159823099773
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,1,0.0074986666440963745
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,1,0.010911999891201654
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,32767,0.7364266713460287
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,7,0.0069866664707660675
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,7,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,15,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,15,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,31,0.007327999919652939
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,63,0.00797333319981893
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,31,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,63,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,127,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,255,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,511,0.008357333640257517
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,511,0.017407999684413273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,1023,0.01145600030819575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,1023,0.018090666582187016
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,2047,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,2047,0.01911466692884763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,4095,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,4095,0.03242666771014532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,8191,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,8191,0.05276266733805338
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,16383,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,16383,0.09489066402117412
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,32767,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,32767,0.1776640017827352
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,65535,0.019797333826621372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,65535,0.3304106593132019
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,131071,0.027818667391935985
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,131071,0.6359039942423502
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,3,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,7,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,15,0.007205333560705185
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,15,0.011424000064531961
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,31,0.007333333293596904
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,31,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,63,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,63,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,127,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,127,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,255,0.00933333362142245
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,255,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,511,0.008474666625261307
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,511,0.017279999951521557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,1023,0.011450666934251785
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,1023,0.019968000551064808
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,2047,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,2047,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,4095,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,4095,0.031231999397277832
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,8191,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,8191,0.05294933418432871
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,16383,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,16383,0.09506133198738098
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,32767,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,32767,0.1776640017827352
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,65535,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,65535,0.3300693432490031
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,131071,0.040106666584809623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,131071,0.6360746622085571
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,1,0.007146666447321574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,1,0.010949333508809408
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,7,0.007146666447321574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,7,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,15,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,15,0.011594666788975397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,31,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,31,0.012128000458081564
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,63,0.008005333443482717
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,63,0.013530666629473368
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,127,0.008389333263039589
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,127,0.013669333110253016
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,255,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,255,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,511,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,511,0.01791999985774358
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,1023,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,1023,0.01854933301607768
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,2047,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,2047,0.018432000031073887
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,4095,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,4095,0.031231999397277832
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,8191,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,8191,0.05328000088532766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,16383,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,16383,0.0951039989789327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,32767,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,32767,0.17749333381652832
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,65535,0.031061333914597828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,65535,0.3300693432490031
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,1,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,131071,0.05563733478387197
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,131071,0.6365866661071777
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,3,0.007151999821265538
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,7,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,15,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,15,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,31,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,31,0.011765333513418833
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,63,0.00785600021481514
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,63,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,127,0.008186666915814081
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,127,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,255,0.00821333316465219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,255,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,511,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,511,0.01791999985774358
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,1023,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,1023,0.018432000031073887
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,2047,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,2047,0.018432000031073887
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,4095,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,4095,0.032261334359645844
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,8191,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,8191,0.054272000988324486
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,16383,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,16383,0.09559999903043111
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,32767,0.03310933212439219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,32767,0.1781760056813558
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,65535,0.3304106593132019
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,65535,0.052906667192777
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,131071,0.2106026609738668
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,131071,0.637610673904419
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,3,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,7,0.007146666447321574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,7,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,15,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,15,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,31,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,31,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,63,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,63,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,255,0.008378666515151659
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,127,0.013669333110253016
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,255,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,511,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,511,0.01791999985774358
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,1023,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,1023,0.0191040001809597
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,2047,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,2047,0.03299200038115183
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,4095,0.053930665055910744
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,4095,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,8191,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,8191,0.09130666653315227
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,16383,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,16383,0.1570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,32767,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,32767,0.2927466630935669
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,1,0.007029333462317784
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,65535,0.021162666380405426
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,65535,0.5758293469746908
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,1,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,131071,0.040789333482583366
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,131071,0.7207252979278564
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,3,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,3,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,7,0.007151999821265538
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,7,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,15,0.011546666423479715
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,15,0.007135999699433644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,31,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,31,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,63,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,63,0.01394133393963178
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,127,0.009258666386206945
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,127,0.01369599997997284
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,255,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,255,0.015024000157912573
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,511,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,511,0.017407999684413273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,1023,0.011946666985750198
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,1023,0.0194560003777345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,2047,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,2047,0.03242666771014532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,4095,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,8191,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,4095,0.05407466491063436
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,8191,0.09131200114885966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,16383,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,32767,0.024746666351954143
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,16383,0.15735466281572977
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,32767,0.29315733909606934
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,65535,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,65535,0.5767093499501547
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,131071,0.05597866574923197
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,1,0.007018666714429855
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,131071,0.7208960056304932
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,3,0.007189333438873291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,15,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,15,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,31,0.007178666690985362
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,31,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,63,0.007877333089709282
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,63,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,127,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,127,0.013658666362365087
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,255,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,255,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,511,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,511,0.020101333657900494
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,1023,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,1023,0.018624000251293182
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,2047,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,2047,0.03276800115903219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,4095,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,4095,0.054101333022117615
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,8191,0.0170666662355264
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,8191,0.09130666653315227
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,16383,0.020992000897725422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,16383,0.15753066539764404
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,32767,0.03328000009059906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,32767,0.29371732473373413
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,65535,0.054101333022117615
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,65535,0.577023983001709
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,131071,0.2104319930076599
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,131071,0.7215786774953207
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,3,0.011087999989589056
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,7,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,7,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,15,0.007173333317041397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,15,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,31,0.007391999786098798
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,31,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,63,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,63,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,127,0.008352000266313553
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,127,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,255,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,255,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,511,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,511,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,1023,0.018789333601792652
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,1023,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,2047,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,2047,0.03242666771014532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,4095,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,8191,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,4095,0.05407466491063436
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,8191,0.09147733449935913
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,16383,0.034474665919939675
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,16383,0.15769599874814352
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,32767,0.05444266895453135
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,32767,0.2935466567675273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,65535,0.2044586737950643
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,65535,0.5771946509679159
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,1,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,131071,0.4020906686782837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,131071,0.7430826822916666
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,15,0.007658666620651881
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,31,0.007290666922926903
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,63,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,127,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,255,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,255,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,511,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,511,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,1023,0.01826133330663045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,1023,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,2047,0.027477333943049114
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,2047,0.024746666351954143
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,4095,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,4095,0.04113066693147024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,8191,0.0576853354771932
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,8191,0.07406933108965556
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,1,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,3,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,16383,0.20428800582885742
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,16383,0.13960533340771994
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,15,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,31,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,63,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,127,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,255,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,255,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,511,0.018090666582187016
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,511,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,1023,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,1023,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,2047,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,2047,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,4095,0.05734399954477946
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,4095,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,8191,0.07389866809050243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,8191,0.2027519941329956
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,1,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,16383,0.1950719952583313
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,16383,0.392192006111145
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,1,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,3,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,7,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,7,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,15,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,15,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,31,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,31,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,63,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,63,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,127,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,127,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,255,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,255,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,511,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,511,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,1023,0.03908266623814901
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,1023,0.025600001215934753
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,2047,0.056661332647005715
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,4095,0.20206934213638306
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,4095,0.07645866771539052
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,8191,0.39458131790161133
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,8191,0.19490132729212442
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,1,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,16383,0.7811413606007894
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,1,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,16383,0.37410132090250653
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,3,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,3,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,7,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,7,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,2047,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,15,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,15,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,31,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,31,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,63,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,63,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,127,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,255,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,255,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,511,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,511,0.032085334261258446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,1023,0.05734399954477946
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,1023,0.04625066618124644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,2047,0.20138667027155557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,2047,0.08055466910203297
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,4095,0.3942399819691976
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,4095,0.19746132691701254
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,8191,0.7814826965332031
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,8191,0.37666134039560956
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,127,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,1,0.007322666545708974
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,16383,0.7348906993865967
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,16383,1.5564799308776855
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,3,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,7,0.0069866664707660675
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,15,0.007189333438873291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,15,0.01090666651725769
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,31,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,31,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,63,0.00816000004609426
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,63,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,255,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,255,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,511,0.017722666263580322
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,511,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,1023,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,1023,0.03379199902216593
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,2047,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,2047,0.05222400029500326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,4095,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,4095,0.07521066566308339
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,8191,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,8191,0.13550933202107748
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,16383,0.24934399127960205
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,32767,0.025770666698614757
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,32767,0.4843519926071167
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,65535,0.03293866664171219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,65535,0.7186773618062338
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,131071,0.056832000613212585
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,131071,1.43394136428833
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,16383,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,3,0.007327999919652939
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,15,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,7,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,15,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,31,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,63,0.008047999814152718
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,63,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,127,0.0084906667470932
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,255,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,255,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,511,0.009056000038981438
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,511,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,1023,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,31,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,1023,0.032255999743938446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,2047,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,2047,0.04863999783992767
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,4095,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,4095,0.07543466488520305
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,8191,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,8191,0.1360213359196981
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,16383,0.24985599517822266
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,32767,0.0341333324710528
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,32767,0.4843306541442871
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,65535,0.05478399991989136
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,65535,0.7195306619008383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,131071,0.20957867304484049
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,1,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,16383,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,131071,1.43394136428833
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,3,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,7,0.012432000289360682
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,15,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,15,0.011946666985750198
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,31,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,63,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,63,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,255,0.008661333471536636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,255,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,511,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,511,0.019626667102177937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,1023,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,1023,0.03242666771014532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,31,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,2047,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,2047,0.048469334840774536
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,4095,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,4095,0.07560533285140991
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,8191,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,8191,0.1358506679534912
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,16383,0.03601066768169403
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,32767,0.05461333195368449
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,32767,0.48469332853953045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,65535,0.20736000935236612
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,65535,0.7200427055358887
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,1,0.011045332998037338
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,131071,0.40294400850931805
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,16383,0.250709335009257
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,131071,1.4641493161519368
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,3,0.007296000296870868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,3,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,7,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,15,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,15,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,31,0.011946666985750198
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,63,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,63,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,127,0.009328000247478485
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,127,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,255,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,255,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,511,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,511,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,1023,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,31,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,1023,0.03260799994071325
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,2047,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,2047,0.04898133377234141
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,4095,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,4095,0.07577600081761678
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,8191,0.13583466410636902
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,8191,0.03737599899371465
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,16383,0.25006399552027386
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,32767,0.20241065820058188
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,32767,0.4848639965057373
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,65535,0.3957759936650594
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,65535,0.7396533489227295
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,16383,0.059562668204307556
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,131071,0.7799466451009115
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,1,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,131071,1.474405288696289
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,1,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,3,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,15,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,15,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,31,0.007791999727487564
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,63,0.007477333148320516
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,63,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,127,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,7,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,255,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,511,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,511,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,1023,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,1023,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,2047,0.04113066693147024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,4095,0.0576853354771932
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,4095,0.041984001795450844
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,8191,0.07441066702206929
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,8191,0.2058239976565043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,1,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,1,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,3,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,3,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,7,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,2047,0.025258667767047882
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,7,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,15,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,31,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,31,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,63,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,63,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,127,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,255,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,255,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,511,0.02918400118748347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,511,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,1023,0.03908266623814901
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,1023,0.025941332181294758
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,2047,0.05751466751098633
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,2047,0.04437333345413208
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,127,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,4095,0.20377600193023682
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,4095,0.0769706666469574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,8191,0.39321601390838623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,1,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,1,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,8191,0.19541333119074503
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,3,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,7,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,7,0.017407999684413273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,15,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,15,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,31,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,31,0.016895999511082966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,63,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,3,0.016895999511082966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,63,0.017237332959969837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,127,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,255,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,127,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,255,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,511,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,511,0.0315733328461647
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,1023,0.04659200211366018
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,2047,0.20411733786265054
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,2047,0.08089600006739299
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,4095,0.3969706694285075
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,4095,0.1960960030555725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,1023,0.05751466751098633
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,1,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,8191,0.785749355951945
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,8191,0.37768534819285077
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,1,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,3,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,7,0.027818667391935985
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,3,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,15,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,15,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,31,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,31,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,63,0.027818667391935985
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,63,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,127,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,7,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,127,0.03601066768169403
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,255,0.03703466554482778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,255,0.042837331692377724
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,511,0.053077335158983864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,511,0.05751466751098633
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,1023,0.20087466637293497
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,1023,0.08635733524958293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,2047,0.1996799906094869
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,4095,0.7847253481547037
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,4095,0.3863893349965413
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,8191,1.5667200088500977
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,1,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,1,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,8191,0.7567359606424967
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,3,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,2047,0.3974826733271281
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,3,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,7,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,7,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,15,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,15,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,31,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,31,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,63,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,127,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,127,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,255,0.018944000204404194
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,511,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,255,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,63,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,511,0.01791999985774358
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,1023,0.03925333420435587
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,1023,0.02628266563018163
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,2047,0.05751466751098633
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,1,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,1,0.017237332959969837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,2047,0.04437333345413208
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,3,0.017407999684413273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,7,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,7,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,3,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,15,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,15,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,31,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,63,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,31,0.016895999511082966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,63,0.017237332959969837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,127,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,255,0.030378667016824085
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,255,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,511,0.040618665516376495
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,511,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,1023,0.05751466751098633
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,1023,0.04693333307902018
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,127,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,2047,0.20514132579167685
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,1,0.028160000840822857
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,1,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,2047,0.08089600006739299
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,3,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,3,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,7,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,15,0.028330666323502857
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,15,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,31,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,63,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,31,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,63,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,7,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,127,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,127,0.03583999971548716
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,255,0.03703466554482778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,255,0.04249600072701772
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,511,0.053077335158983864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,511,0.056661332647005715
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,1023,0.202239990234375
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,2047,0.4007253249486287
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,1,0.05034666756788889
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,2047,0.20070399840672812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,1023,0.08703999718030293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,1,0.05034666756788889
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,3,0.05034666756788889
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,3,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,7,0.05034666756788889
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,7,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,15,0.05034666756788889
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,15,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,31,0.05034666756788889
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,63,0.05034666756788889
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,63,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,127,0.05034666756788889
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,31,0.05034666756788889
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,127,0.06382933259010315
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,255,0.06570666531721751
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,255,0.07833600044250488
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,511,0.19797333081563315
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,511,0.10478933652242024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,1023,0.39202133814493817
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,1023,0.19404800732930502
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,1,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,2047,0.377344012260437
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,1,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,3,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,3,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,7,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,7,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,2047,0.7767039934794108
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,15,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,15,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,31,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,31,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,63,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,127,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,63,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,255,0.031061333914597828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,255,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,511,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,127,0.021503999829292297
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,511,0.03293866664171219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,1023,0.057855998476346336
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,1,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,1023,0.048298666874567665
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,1,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,3,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,3,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,7,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,15,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,15,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,31,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,7,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,31,0.029696000119050343
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,63,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,63,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,127,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,127,0.0365226666132609
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,255,0.0363520011305809
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,255,0.04351999859015147
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,511,0.05853866537412008
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,1023,0.20497065782546997
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,511,0.05273599922657013
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,1,0.05034666756788889
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,1,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,1023,0.08823466300964355
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,3,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,3,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,7,0.05034666756788889
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,7,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,15,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,31,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,31,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,63,0.05034666756788889
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,15,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,63,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,127,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,127,0.06468266745408376
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,255,0.06587733328342438
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,255,0.07901866734027863
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,511,0.20121600230534872
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,511,0.10478933652242024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,1023,0.19694934288660684
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,1,0.09335466225941975
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,1,0.09540266791979472
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,3,0.09352533022562663
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,1023,0.39611732959747314
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,3,0.09540266791979472
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,7,0.09352533022562663
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,7,0.09540266791979472
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,15,0.09352533022562663
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,15,0.09540266791979472
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,31,0.09352533022562663
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,31,0.09540266791979472
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,63,0.09540266791979472
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,127,0.09437867005666097
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,63,0.09352533022562663
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,127,0.12185600399971008
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,255,0.1971199909845988
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,255,0.15069866180419922
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,511,0.21589332818984985
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,511,0.38894931475321454
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,1023,0.7729492982228597
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,1,0.010901333143313726
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,1023,0.37512532869974774
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,3,0.0075040000180403394
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,1,0.007658666620651881
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,7,0.010869332899649939
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,15,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,15,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,31,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,31,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,63,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,127,0.013525333255529404
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,63,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,255,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,255,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,511,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,511,0.031061333914597828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,1023,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,1023,0.04372799893220266
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,2047,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,4095,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,4095,0.1088853379090627
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,8191,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,2047,0.0653653343518575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,8191,0.19592533508936563
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,16383,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,16383,0.36983466148376465
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,32767,0.034474665919939675
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,1,0.007360000163316727
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,32767,0.7188639640808105
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,65535,0.0554666668176651
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,65535,1.4306559562683105
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,1,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,3,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,131071,2.8584960301717124
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,131071,0.2111146648724874
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,3,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,7,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,7,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,15,0.00731733317176501
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,31,0.007631999750932057
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,31,0.01211200033624967
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,63,0.01422400027513504
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,127,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,255,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,255,0.014698666830857595
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,511,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,511,0.031231999397277832
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,1023,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,1023,0.04369066655635834
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,2047,0.06553600231806438
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,4095,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,4095,0.10871466994285583
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,2047,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,8191,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,8191,0.19595734278361002
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,16383,0.03583999971548716
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,16383,0.3701759974161784
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,32767,0.05461333195368449
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,32767,0.7188479900360107
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,65535,0.2058239976565043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,1,0.0074560002734263735
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,131071,0.4010666608810425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,65535,1.433626651763916
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,3,0.007184000064929326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,131071,2.9023574193318686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,7,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,7,0.011637333780527115
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,15,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,31,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,31,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,63,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,63,0.0141546664138635
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,255,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,511,0.0314026673634847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,1023,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,1023,0.04390400151411692
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,2047,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,511,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,2047,0.06524266799290974
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,4095,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,4095,0.10854400197664897
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,8191,0.037205333511034645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,8191,0.1963040033976237
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,16383,0.058880001306533813
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,16383,0.37035199006398517
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,32767,0.2032639980316162
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,32767,0.719871997833252
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,65535,0.3947519858678182
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,65535,1.459712028503418
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,1,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,131071,0.7826773325602213
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,3,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,131071,2.911402702331543
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,3,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,7,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,15,0.007471999774376552
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,15,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,31,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,31,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,63,0.007477333148320516
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,127,0.007482666522264481
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,255,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,127,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,63,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,511,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,511,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,1023,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,1023,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,2047,0.02679466704527537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,2047,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,4095,0.040789333482583366
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,4095,0.040448000033696495
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,8191,0.05751466751098633
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,8191,0.07287466526031494
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,16383,0.20087466637293497
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,16383,0.13909332950909933
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,32767,0.3935573498408
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,32767,0.27187200387318927
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,65535,0.7801173528035482
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,65535,0.5338453451792399
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,131071,1.5496533711751301
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,1,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,1,0.03089066594839096
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,131071,1.0605226357777913
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,3,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,3,0.03089066594839096
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,7,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,7,0.031061333914597828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,15,0.03089066594839096
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,15,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,31,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,31,0.031061333914597828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,63,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,63,0.03089066594839096
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,127,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,255,0.03788800040880839
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,1,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,255,0.04642133414745331
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,127,0.038058665891488395
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,1,0.05239466826121012
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,511,0.054272000988324486
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,511,0.061610668897628784
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,3,0.051541333397229515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,3,0.05239466826121012
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,7,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,7,0.05256533126036326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,15,0.05256533126036326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,31,0.051541333397229515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,15,0.051541333397229515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,31,0.05239466826121012
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,63,0.051541333397229515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,63,0.05239466826121012
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,127,0.051541333397229515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,127,0.06638933221499126
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,255,0.06690133114655812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,255,0.08004266520341237
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,1,0.09454933802286784
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,511,0.10752000411351521
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,511,0.2070186734199524
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,1,0.09642666578292847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,3,0.09437867005666097
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,3,0.09642666578292847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,7,0.09454933802286784
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,7,0.09642666578292847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,15,0.09454933802286784
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,15,0.09642666578292847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,31,0.09642666578292847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,31,0.09454933802286784
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,63,0.09437867005666097
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,63,0.09642666578292847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,127,0.09523199995358785
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,127,0.1230506698290507
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,255,0.20377600193023682
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,255,0.15172266960144043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,1,0.18363734086354574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,1,0.18551466862360635
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,511,0.39611732959747314
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,3,0.18363734086354574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,3,0.18551466862360635
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,511,0.21760000785191855
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,7,0.18363734086354574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,7,0.18466132879257202
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,15,0.18346667289733887
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,15,0.18551466862360635
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,31,0.18363734086354574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,31,0.18551466862360635
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,63,0.18363734086354574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,127,0.2501973311106364
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,63,0.18568533658981323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,127,0.23705599705378214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,255,0.3031040032704671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,255,0.3974826733271281
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,511,0.7661226590474447
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,511,0.4293973445892334
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,float16,1,0.05341866612434387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,fp8,1,0.05461333195368449
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,float16,3,0.05358933409055074
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,fp8,3,0.05461333195368449
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,float16,7,0.05341866612434387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,fp8,7,0.05461333195368449
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,float16,15,0.05341866612434387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,fp8,15,0.05461333195368449
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,float16,31,0.05341866612434387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,fp8,31,0.05461333195368449
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,float16,63,0.05341866612434387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,fp8,63,0.05461333195368449
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,float16,127,0.05358933409055074
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,fp8,127,0.06877866884072621
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,float16,255,0.06894933183987935
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,float16,1,0.09659733374913533
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,fp8,1,0.09864532947540283
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,fp8,255,0.08482133348782857
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,fp8,3,0.09864532947540283
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,float16,3,0.09659733374913533
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,float16,7,0.09659733374913533
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,fp8,7,0.09864532947540283
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,float16,15,0.09642666578292847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,fp8,15,0.09864532947540283
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,float16,31,0.09659733374913533
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,fp8,31,0.09864532947540283
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,float16,63,0.09659733374913533
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,float16,127,0.09693866968154907
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,fp8,63,0.09898666540781657
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,fp8,127,0.1262933313846588
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,float16,1,0.18568533658981323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,float16,255,0.212991992632548
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,fp8,255,0.15377066532770792
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,fp8,1,0.18773333231608072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,float16,3,0.18568533658981323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,fp8,3,0.18773333231608072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,fp8,7,0.18807466824849448
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,float16,7,0.18568533658981323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,float16,15,0.18568533658981323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,fp8,15,0.18773333231608072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,float16,31,0.18568533658981323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,fp8,31,0.18773333231608072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,float16,63,0.18602667252222696
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,fp8,63,0.1879040002822876
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,float16,127,0.25514666239420575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,fp8,127,0.23995733261108398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,float16,1,0.36420265833536786
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,fp8,255,0.3083946704864502
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,float16,255,0.41096532344818115
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,fp8,1,0.36556800206502277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,float16,3,0.36420265833536786
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,fp8,3,0.36556800206502277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,float16,7,0.36403199036916095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,fp8,7,0.36556800206502277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,float16,15,0.36454399426778156
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,fp8,15,0.3660800059636434
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,float16,31,0.36471466223398846
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,fp8,31,0.36522666613260907
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,float16,63,0.452949325243632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,fp8,63,0.36556800206502277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,1,0.006826666494210561
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,1,0.01911466692884763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,float16,127,0.4843519926071167
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,3,0.019626667102177937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,3,0.006655999769767125
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,fp8,127,0.5205333232879639
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,fp8,255,0.6045013268788656
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,float16,255,0.7959893544514974
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,7,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,7,0.019626667102177937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,15,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,15,0.019968000551064808
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,31,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,31,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,63,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,63,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,127,0.009098666409651438
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,127,0.024746666351954143
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,255,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,255,0.025429333249727886
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,511,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,1023,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,511,0.030405332644780476
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,1023,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,2047,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,2047,0.09454933802286784
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,4095,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,4095,0.18039466937383017
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,8191,0.024405332903067272
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,8191,0.3517013390858968
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,16383,0.695466677347819
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,32767,0.055125330885251365
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,16383,0.03549866626660029
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,32767,1.431909402211507
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,1,0.006655999769767125
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,65535,0.20684800545374551
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,1,0.019573333362738293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,65535,2.8540267944335938
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,3,0.006810666372378667
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,3,0.019258666783571243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,7,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,7,0.019285333653291065
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,15,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,15,0.01911466692884763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,31,0.007637333124876022
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,31,0.04539733131726583
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,63,0.008858666444818178
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,63,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,127,0.024746666351954143
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,127,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,255,0.04727466901143392
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,511,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,511,0.030720000465710957
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,1023,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,1023,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,2047,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,2047,0.09471999605496724
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,4095,0.025429333249727886
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,4095,0.17971199750900269
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,8191,0.03737599899371465
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,8191,0.3520853519439697
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,16383,0.05870933334032694
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,16383,0.6966613133748373
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,32767,0.2034346659978231
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,1,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,65535,0.39321601390838623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,32767,1.4347947438557942
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,65535,2.8984320958455405
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,1,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,3,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,7,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,15,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,31,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,15,0.00919999989370505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,31,0.008661333471536636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,63,0.007285333548982938
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,63,0.008522666369875273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,127,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,127,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,255,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,1023,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,511,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,511,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,1023,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,2047,0.026965332527955372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,2047,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,4095,0.04095999896526337
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,4095,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,8191,0.0576853354771932
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,8191,0.0730453332265218
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,16383,0.13875200351079306
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,16383,0.20258132616678873
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,32767,0.39509332180023193
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,32767,0.2717013359069824
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,65535,0.7782399654388428
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,65535,0.5338453451792399
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,3,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,15,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,31,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,63,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,63,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,127,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,255,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,255,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,511,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,511,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,1023,0.02867199977238973
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,1023,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,2047,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,2047,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,4095,0.056320001681645714
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,4095,0.04095999896526337
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,8191,0.20138667027155557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,8191,0.07355733215808868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,16383,0.3935573498408
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,16383,0.19473065932591757
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,32767,0.7821653683980306
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,32767,0.3752959966659546
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,1,0.01028266673286756
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,1,0.022863999009132385
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,65535,1.5612586339314778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,65535,0.7396693229675293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,3,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,3,0.021333334346612293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,7,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,7,0.02199999988079071
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,15,0.011391999820868174
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,15,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,31,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,31,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,63,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,127,0.029696000119050343
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,63,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,255,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,255,0.051029334465662636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,511,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,511,0.09454933802286784
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,1023,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,1023,0.18227199713389078
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,2047,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,2047,0.35737065474192303
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,4095,0.0339626669883728
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,4095,0.7070720195770264
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,8191,0.05034666756788889
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,8191,1.4269439379374187
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,16383,0.22272000710169473
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,32767,0.4278613328933716
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,1,0.00816000004609426
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,16383,4.238335927327474
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,3,0.007978666573762894
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,3,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,7,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,32767,9.013226826985678
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,15,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,31,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,31,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,63,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,255,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,255,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,511,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,511,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,1023,0.022015998760859173
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,1023,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,2047,0.03515733281771342
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,2047,0.04215466479460398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,4095,0.07509333391984303
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,8191,0.21657600005467734
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,4095,0.05751466751098633
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,8191,0.14062933127085367
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,16383,0.4230826695760091
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,16383,0.2757973273595174
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,1,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,1,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,32767,0.8389973640441895
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,32767,0.5381120045979818
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,31,0.008186666915814081
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,63,0.008496000121037165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,31,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,63,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,127,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,255,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,255,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,511,0.022357332209746044
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,511,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,1023,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,1023,0.03549866626660029
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,2047,0.05461333195368449
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,2047,0.04232533276081085
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,4095,0.2167466680208842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,4095,0.07492266595363617
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,8191,0.4237653414408366
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,8191,0.2152106761932373
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,16383,0.8372906843821207
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,16383,0.4107946554819743
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,32767,1.6708265940348308
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,32767,0.8017919858296713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,1,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,1,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,3,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,3,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,7,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,7,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,15,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,31,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,15,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,31,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,127,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,127,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,255,0.022015998760859173
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,511,0.02679466704527537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,1023,0.05239466826121012
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,1023,0.043178667624791466
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,2047,0.21401600042978922
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,63,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,2047,0.07799466451009114
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,63,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,4095,0.4215466578801473
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,255,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,511,0.0363520011305809
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,4095,0.21657600005467734
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,8191,0.4130133390426636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,8191,0.8326826890309652
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,16383,0.8093012968699137
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,1,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,3,0.007301333049933116
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,1,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,3,0.011946666985750198
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,32767,3.3416531880696616
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,32767,1.5977813402811687
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,7,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,7,0.012149333953857422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,15,0.0075040000180403394
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,31,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,15,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,16383,1.665877342224121
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,31,0.014479999740918478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,63,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,127,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,127,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,255,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,255,0.018629333625237148
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,511,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,511,0.020138667275508244
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,1023,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,1023,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,2047,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,2047,0.032255999743938446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,4095,0.021562665700912476
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,8191,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,4095,0.05444266895453135
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,8191,0.09659733374913533
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,16383,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,16383,0.1812480092048645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,32767,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,32767,0.33604268232981366
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,65535,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,65535,0.6446239948272705
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,1,0.007029333462317784
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,1,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,131071,0.03532800078392029
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,131071,1.2688799699147542
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,3,0.007141333073377609
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,3,0.013621332744757334
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,7,0.007130666946371396
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,7,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,15,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,31,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,31,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,63,0.015194666882356008
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,127,0.015552000453074774
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,255,0.008389333263039589
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,255,0.01911466692884763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,511,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,511,0.020138667275508244
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,1023,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,1023,0.020992000897725422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,2047,0.013183999806642532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,2047,0.034671999514102936
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,4095,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,4095,0.054272000988324486
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,8191,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,8191,0.09676800171534221
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,16383,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,16383,0.18107734123865762
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,32767,0.021333334346612293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,32767,0.33642133076985675
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,65535,0.028501334289709728
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,65535,0.6451199849446615
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,1,0.007173333317041397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,131071,0.05358933409055074
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,1,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,131071,1.269759972890218
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,3,0.007157333195209503
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,3,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,7,0.007141333073377609
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,7,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,15,0.00766933336853981
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,31,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,15,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,31,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,63,0.008400000010927519
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,127,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,127,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,255,0.008383999889095625
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,255,0.020138667275508244
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,511,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,511,0.020138667275508244
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,1023,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,1023,0.020938667158285778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,2047,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,2047,0.03242666771014532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,4095,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,4095,0.054272000988324486
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,8191,0.015872000406185787
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,8191,0.09715732932090759
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,16383,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,16383,0.18094933032989502
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,32767,0.0314026673634847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,32767,0.33655468622843426
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,65535,0.04334933559099833
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,65535,0.6449493169784546
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,1,0.007184000064929326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,1,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,131071,0.23278933763504028
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,3,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,131071,1.2698079744974773
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,7,0.00730666642387708
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,7,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,15,0.007471999774376552
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,15,0.014197333405415217
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,31,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,31,0.015024000157912573
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,127,0.008330666770537695
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,127,0.015696000307798386
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,255,0.008394666636983553
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,255,0.020138667275508244
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,511,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,511,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,1023,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,1023,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,2047,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,4095,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,4095,0.05580799778302511
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,8191,0.09796266754468282
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,16383,0.027136000494162243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,16383,0.18244266510009766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,32767,0.04437333345413208
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,32767,0.3375786542892456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,65535,0.22869332631429037
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,65535,0.6461439927419027
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,2047,0.03363733241955439
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,131071,0.446122686068217
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,8191,0.02218666672706604
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,1,0.013280000537633896
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,131071,1.3044053713480632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,3,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,7,0.007034666836261749
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,7,0.012159999459981918
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,15,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,31,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,31,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,63,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,127,0.015568000574906668
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,255,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,511,0.011941333611806234
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,255,0.0194560003777345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,511,0.021333334346612293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,1023,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,1023,0.03532800078392029
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,2047,0.02218666672706604
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,2047,0.05514133473237356
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,4095,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,4095,0.09318400422732036
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,8191,0.01621333385507266
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,8191,0.16059733430544534
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,16383,0.017237332959969837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,16383,0.2995199958483378
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,32767,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,32767,0.5853866736094157
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,65535,0.02867199977238973
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,65535,1.152511994043986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,1,0.007178666690985362
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,131071,0.054101333022117615
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,1,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,3,0.007162666569153468
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,3,0.013002666334311167
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,131071,1.4373547236124675
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,7,0.007157333195209503
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,7,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,15,0.007344000041484833
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,31,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,15,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,63,0.008223999912540117
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,31,0.014885333677132925
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,63,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,127,0.008170666793982187
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,127,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,255,0.008367999767263731
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,255,0.019626667102177937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,511,0.020629333953062694
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,1023,0.013477332890033722
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,511,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,1023,0.03498666733503342
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,2047,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,2047,0.05563733478387197
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,4095,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,4095,0.09352533022562663
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,8191,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,8191,0.16076800227165222
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,16383,0.2995199958483378
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,16383,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,32767,0.032255999743938446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,32767,0.5860693454742432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,65535,0.04471466441949209
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,65535,1.153872013092041
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,131071,0.233130673567454
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,1,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,131071,1.4383412996927898
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,3,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,7,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,15,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,31,0.0081386665503184
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,31,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,63,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,63,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,127,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,255,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,255,0.019978666057189304
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,511,0.011941333611806234
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,1023,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,511,0.020634666085243225
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,1023,0.0339626669883728
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,2047,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,2047,0.05597866574923197
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,4095,0.09403733412424724
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,4095,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,8191,0.02218666672706604
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,8191,0.1609386702378591
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,16383,0.2998613317807515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,16383,0.027647999425729115
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,32767,0.04369066655635834
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,32767,0.5864106814066569
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,65535,0.22818134228388467
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,65535,1.1537386576334636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,131071,0.4457813501358032
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,1,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,131071,1.462272008260091
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,3,0.007135999699433644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,3,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,7,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,15,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,31,0.008170666793982187
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,31,0.015872000406185787
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,63,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,127,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,127,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,255,0.0194560003777345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,511,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,511,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,1023,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,1023,0.0339626669883728
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,2047,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,2047,0.055626665552457176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,4095,0.019968000551064808
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,4095,0.09403199950853984
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,8191,0.031061333914597828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,16383,0.04505600035190582
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,8191,0.1616213321685791
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,16383,0.3002026677131653
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,32767,0.22203733523686728
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,32767,0.5865813493728638
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,65535,0.4365653196970622
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,65535,1.18340269724528
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,1,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,131071,0.8671573003133138
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,131071,1.4842880566914876
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,7,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,7,0.009375999992092451
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,15,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,15,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,31,0.008474666625261307
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,63,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,127,0.008527999743819237
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,63,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,255,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,511,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,255,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,511,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,1023,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,1023,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,2047,0.03515733281771342
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,2047,0.04215466479460398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,4095,0.05751466751098633
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,4095,0.07492266595363617
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,8191,0.14062933127085367
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,8191,0.22033067544301352
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,1,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,16383,0.42956801255544025
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,16383,0.2752853234608968
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,3,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,7,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,15,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,31,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,63,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,127,0.00949866697192192
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,255,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,255,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,511,0.022357332209746044
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,511,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,1023,0.03601066768169403
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,1023,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,2047,0.0554666668176651
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,2047,0.04232533276081085
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,4095,0.07509333391984303
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,4095,0.21794132391611734
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,8191,0.21640533208847046
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,8191,0.4235946734746297
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,1,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,1,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,16383,0.41710933049519855
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,16383,0.8340480327606201
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,3,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,7,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,3,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,7,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,15,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,15,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,31,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,31,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,63,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,63,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,127,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,127,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,255,0.022015998760859173
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,255,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,511,0.03601066768169403
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,511,0.026965332527955372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,1023,0.05205333232879639
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,1023,0.043178667624791466
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,2047,0.21504000822703043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,2047,0.07816533247629802
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,4095,0.420693318049113
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,4095,0.21845332781473795
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,8191,0.8314879735310873
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,8191,0.4169386625289917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,1,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,16383,1.6629759470621746
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,1,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,16383,0.8209066390991211
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,3,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,3,0.019797333826621372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,7,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,7,0.019968000551064808
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,15,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,31,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,31,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,15,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,63,0.017407999684413273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,63,0.019968000551064808
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,127,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,127,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,255,0.03669333209594091
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,255,0.029525332152843475
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,511,0.05376000205675761
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,511,0.049322664737701416
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,1023,0.21128533283869425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,1023,0.0820906658967336
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,2047,0.42001068592071533
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,2047,0.22016000747680664
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,4095,0.8337066968282064
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,4095,0.4126720031102498
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,8191,1.665877342224121
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,8191,0.80076797803243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,1,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,16383,1.5802027384440105
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,16383,3.3344853719075522
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,3,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,7,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,15,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,31,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,31,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,63,0.008394666636983553
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,255,0.008858666444818178
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,255,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,511,0.012602667013804117
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,511,0.0341333324710528
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,1023,0.022405333817005157
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,1023,0.04983466863632202
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,2047,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,2047,0.07770666480064392
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,4095,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,4095,0.13943466544151306
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,8191,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,8191,0.2555039922396342
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,16383,0.019285333653291065
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,16383,0.49186134338378906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,32767,0.03310933212439219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,32767,0.9605120023091634
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,65535,0.04539733131726583
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,65535,1.4351360003153484
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,1,0.007194666812817256
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,1,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,131071,0.22988800207773843
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,3,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,131071,2.8675308227539062
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,3,0.01259200026591619
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,15,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,7,0.007525333513816197
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,7,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,31,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,31,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,63,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,127,0.008416000132759413
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,127,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,255,0.019285333653291065
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,255,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,511,0.012650666137536367
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,511,0.034304000437259674
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,1023,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,1023,0.0506879985332489
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,2047,0.07768533130486806
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,2047,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,4095,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,4095,0.13960533340771994
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,8191,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,8191,0.25563732782999676
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,16383,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,16383,0.4928853511810303
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,32767,0.04539733131726583
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,32767,0.9613653024037679
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,65535,0.2290346622467041
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,65535,1.4365013440450032
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,1,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,131071,0.44492801030476886
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,1,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,131071,2.906794548034668
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,3,0.007296000296870868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,3,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,7,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,15,0.007626666376988093
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,15,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,31,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,63,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,31,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,63,0.014698666830857595
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,255,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,127,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,255,0.020682666450738907
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,511,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,511,0.0341333324710528
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,1023,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,1023,0.0506879985332489
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,2047,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,2047,0.07850666840871175
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,4095,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,4095,0.1389226714769999
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,8191,0.031744000812371574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,8191,0.25563732782999676
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,16383,0.04539733131726583
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,16383,0.4929973284403483
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,32767,0.22374399503072104
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,32767,0.9620479742685953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,65535,0.4369066556294759
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,65535,1.458346684773763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,1,0.012975999464591345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,131071,0.8710827032725016
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,131071,2.916693369547526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,3,0.013280000537633896
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,7,0.007520000139872233
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,7,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,15,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,31,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,31,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,63,0.008367999767263731
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,63,0.015872000406185787
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,127,0.009029333169261614
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,127,0.015013333410024643
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,255,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,255,0.019626667102177937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,511,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,511,0.034474665919939675
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,1023,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,1023,0.05034666756788889
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,2047,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,2047,0.07834666470686595
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,4095,0.03310933212439219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,4095,0.13926399747530618
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,8191,0.04949333270390829
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,8191,0.25619200865427655
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,16383,0.21896533171335855
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,16383,0.49289600054423016
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,32767,0.42905600865681964
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,32767,0.9861120382944742
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,65535,0.8564053376515707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,65535,1.482111930847168
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,1,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,131071,1.7061546643575032
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,131071,2.954239845275879
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,3,0.008687999720374743
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,3,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,7,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,15,0.00847999999920527
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,31,0.008512000242869059
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,63,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,31,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,63,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,255,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,255,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,511,0.023210667073726654
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,511,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,1023,0.037205333511034645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,1023,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,2047,0.05614933371543884
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,2047,0.04266666869322459
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,4095,0.22272000710169473
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,4095,0.07492266595363617
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,1,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,1,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,8191,0.4278613328933716
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,3,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,3,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,8191,0.21691733598709106
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,7,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,7,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,15,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,15,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,31,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,31,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,63,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,127,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,63,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,127,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,255,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,255,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,511,0.036864000062147774
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,511,0.026965332527955372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,1023,0.05273599922657013
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,1023,0.04334933559099833
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,2047,0.21913599967956543
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,2047,0.07833600044250488
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,4095,0.2198186715443929
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,4095,0.4261546532313029
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,1,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,1,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,8191,0.8367786407470703
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,8191,0.4196693499883016
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,3,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,3,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,7,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,7,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,15,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,15,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,31,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,63,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,31,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,63,0.016895999511082966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,127,0.023039999107519787
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,127,0.020992000897725422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,255,0.03703466554482778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,255,0.029696000119050343
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,511,0.05376000205675761
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,511,0.049322664737701416
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,1023,0.08226133386294048
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,1023,0.21555199225743613
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,2047,0.42581331729888916
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,2047,0.22203733523686728
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,4095,0.4116479953130086
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,4095,0.8357546329498291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,1,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,1,0.036864000062147774
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,8191,0.8023040294647217
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,8191,1.66758394241333
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,3,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,3,0.03618133316437403
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,7,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,7,0.03618133316437403
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,15,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,15,0.03703466554482778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,31,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,31,0.03703466554482778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,63,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,63,0.03618133316437403
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,127,0.03703466554482778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,127,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,255,0.04744533201058706
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,255,0.05358933409055074
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,511,0.2141866683959961
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,511,0.09250133236249287
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,1023,0.4264959891637166
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,1023,0.22118399540583292
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,2047,0.846677303314209
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,2047,0.42632532119750977
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,4095,0.8306346734364828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,4095,1.683626651763916
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,1,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,8191,1.6430080731709797
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,1,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,8191,3.366741180419922
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,3,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,3,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,7,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,7,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,15,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,31,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,31,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,63,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,63,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,15,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,127,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,255,0.023039999107519787
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,255,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,511,0.03754666695992152
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,511,0.026965332527955372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,1023,0.05341866612434387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,1023,0.04334933559099833
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,2047,0.22340265909830728
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,2047,0.07901866734027863
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,1,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,1,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,3,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,3,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,7,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,15,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,7,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,15,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,31,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,63,0.017237332959969837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,31,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,63,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,127,0.021333334346612293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,127,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,255,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,511,0.05444266895453135
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,255,0.03737599899371465
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,511,0.05034666756788889
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,1023,0.08243200182914734
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,1023,0.21913599967956543
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,2047,0.43110398451487225
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,1,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,1,0.037205333511034645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,2047,0.22152533133824667
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,3,0.03703466554482778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,7,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,7,0.037205333511034645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,3,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,15,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,15,0.037205333511034645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,31,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,31,0.037205333511034645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,63,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,63,0.03703466554482778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,127,0.037205333511034645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,127,0.04215466479460398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,255,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,255,0.05358933409055074
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,511,0.21657600005467734
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,511,0.09250133236249287
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,1023,0.43110398451487225
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,1023,0.2254506746927897
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,2047,0.8512852986653646
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,1,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,2047,0.4264959891637166
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,1,0.06587733328342438
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,3,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,3,0.06587733328342438
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,7,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,7,0.06570666531721751
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,15,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,15,0.06587733328342438
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,31,0.06587733328342438
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,63,0.0506879985332489
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,31,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,63,0.06587733328342438
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,127,0.06621866424878438
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,127,0.07577600081761678
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,255,0.20906666914621988
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,255,0.09676800171534221
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,511,0.4198400179545085
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,511,0.2106026609738668
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,1023,0.8366080125172933
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,1023,0.41437868277231854
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,2047,0.8082773685455322
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,2047,1.668949286142985
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,1,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,1,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,3,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,3,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,7,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,7,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,15,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,15,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,31,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,31,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,63,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,63,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,127,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,255,0.03908266623814901
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,255,0.030378667016824085
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,127,0.024746666351954143
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,511,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,511,0.05597866574923197
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,1023,0.22579199075698853
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,1023,0.08328533172607422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,1,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,1,0.038058665891488395
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,3,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,7,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,7,0.038058665891488395
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,3,0.038058665891488395
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,15,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,15,0.038058665891488395
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,31,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,31,0.038058665891488395
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,63,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,63,0.03822933385769526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,127,0.03822933385769526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,127,0.04249600072701772
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,255,0.049322664737701416
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,255,0.05444266895453135
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,511,0.22323199113210043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,511,0.09352533022562663
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,1,0.051541333397229515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,1023,0.4394666751225789
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,1023,0.22920533021291098
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,1,0.06690133114655812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,3,0.051541333397229515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,3,0.06690133114655812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,7,0.051541333397229515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,7,0.06673066814740498
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,15,0.051541333397229515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,15,0.06690133114655812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,31,0.051541333397229515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,31,0.06690133114655812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,63,0.051541333397229515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,63,0.06690133114655812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,127,0.06690133114655812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,127,0.07594666878382365
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,255,0.21452800432840982
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,255,0.09779199957847595
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,511,0.4266666571299235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,511,0.2172586719195048
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,1,0.09540266791979472
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,1,0.12424533565839131
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,1023,0.42496001720428467
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,1023,0.8478720188140869
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,3,0.09506133198738098
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,3,0.12424533565839131
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,7,0.09523199995358785
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,15,0.12424533565839131
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,15,0.09540266791979472
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,7,0.12424533565839131
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,31,0.09523199995358785
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,31,0.12424533565839131
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,63,0.09608532985051473
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,63,0.12458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,127,0.22715733448664346
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,127,0.14455466469128928
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,255,0.41250133514404297
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,255,0.21248000860214233
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,511,0.4055039882659912
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,511,0.8241493701934814
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,1023,0.8026453653971354
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,1023,1.657002607981364
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,1,0.012752000242471695
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,3,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,7,0.007365333537260692
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,7,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,3,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,15,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,31,0.008330666770537695
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,31,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,63,0.00873066671192646
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,127,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,255,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,255,0.03293866664171219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,511,0.02145066608985265
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,511,0.04535999894142151
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,1023,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,1023,0.06724266707897186
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,2047,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,2047,0.11195733149846394
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,4095,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,4095,0.20002132654190063
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,8191,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,8191,0.37700267632802326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,16383,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,16383,0.7296106815338135
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,32767,0.04488533238569895
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,65535,0.2244266668955485
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,32767,1.435306708017985
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,65535,2.861226717631022
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,131071,0.4369066556294759
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,1,0.007674666742483775
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,1,0.012597333639860153
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,131071,5.788373311360677
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,3,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,7,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,15,0.007514666765928268
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,15,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,31,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,31,0.014368000129858652
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,63,0.008661333471536636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,63,0.014501333236694336
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,127,0.01451733335852623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,127,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,255,0.009701333319147428
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,255,0.03328000009059906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,511,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,511,0.0455626646677653
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,1023,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,1023,0.06758399804433186
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,2047,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,4095,0.021503999829292297
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,4095,0.20154666900634766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,2047,0.11127466956774394
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,8191,0.032085334261258446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,8191,0.37768534819285077
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,16383,0.045567999283472695
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,16383,0.7309652964274088
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,32767,0.2228906750679016
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,32767,1.4353173573811848
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,65535,0.43537068367004395
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,1,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,65535,2.900634765625
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,131071,0.8630613485972086
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,1,0.012784000486135483
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,3,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,3,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,131071,5.80027707417806
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,7,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,7,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,15,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,31,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,31,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,63,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,127,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,255,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,511,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,511,0.045738667249679565
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,255,0.03310933212439219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,1023,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,1023,0.06758399804433186
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,2047,0.02218666672706604
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,2047,0.11365866661071777
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,4095,0.03328000009059906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,4095,0.20172800620396933
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,8191,0.049322664737701416
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,8191,0.37802668412526447
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,16383,0.22033067544301352
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,16383,0.7312800089518229
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,32767,0.42973868052164715
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,32767,1.458346684773763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,65535,0.8553813298543295
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,1,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,65535,2.910208066304525
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,1,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,131071,1.7017173767089844
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,3,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,131071,5.813418706258138
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,7,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,15,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,15,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,31,0.007978666573762894
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,31,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,63,0.008522666369875273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,127,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,255,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,255,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,511,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,511,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,1023,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,1023,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,2047,0.034645333886146545
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,2047,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,4095,0.056832000613212585
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,4095,0.07406933108965556
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,8191,0.13960533340771994
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,8191,0.2136746644973755
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,16383,0.42239999771118164
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,16383,0.27477333943049115
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,32767,0.8470186392466227
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,32767,0.5369173288345337
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,65535,1.061205307642619
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,65535,1.6897706985473633
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,1,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,131071,2.111999988555908
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,131071,3.362133344014486
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,1,0.040106666584809623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,3,0.032085334261258446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,3,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,7,0.032085334261258446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,7,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,15,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,15,0.040106666584809623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,31,0.032085334261258446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,31,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,63,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,63,0.040106666584809623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,127,0.040448000033696495
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,127,0.04454400142033895
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,255,0.051882664362589516
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,255,0.056661332647005715
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,511,0.23227733373641968
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,1,0.05341866612434387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,511,0.09471999605496724
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,3,0.05341866612434387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,3,0.06911999980608623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,1,0.06894933183987935
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,7,0.05341866612434387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,7,0.06877866884072621
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,15,0.05358933409055074
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,15,0.06877866884072621
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,31,0.05341866612434387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,31,0.06894933183987935
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,63,0.05358933409055074
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,63,0.06877866884072621
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,127,0.06877866884072621
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,127,0.07799466451009114
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,255,0.22459733486175537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,255,0.09983999530474345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,1,0.09659733374913533
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,1,0.1262933313846588
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,511,0.4396373430887858
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,3,0.09642666578292847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,511,0.22698666652043661
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,3,0.1262933313846588
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,7,0.09659733374913533
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,7,0.1262933313846588
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,15,0.09659733374913533
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,31,0.12612266341845194
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,31,0.09659733374913533
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,15,0.1262933313846588
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,63,0.09762133161226909
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,63,0.12612266341845194
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,127,0.2106026609738668
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,127,0.14660267035166422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,255,0.42496001720428467
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,255,0.22101332743962607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,1,0.18568533658981323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,1,0.2450773318608602
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,511,0.839680035909017
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,511,0.4194986820220947
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,3,0.1858560045560201
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,3,0.2450773318608602
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,7,0.18568533658981323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,7,0.2450773318608602
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,15,0.18568533658981323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,15,0.24524799982706705
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,31,0.18773333231608072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,31,0.2450773318608602
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,63,0.2676053245862325
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,63,0.2455893357594808
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,127,0.4020906686782837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,127,0.3089066743850708
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,255,0.8120319843292236
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,255,0.42052265008290607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,511,1.632597287495931
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,511,0.7896746794382731
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,fp8,1,0.0718506673971812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,1,0.056661332647005715
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,fp8,3,0.07202133536338806
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,3,0.056661332647005715
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,7,0.056661332647005715
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,fp8,7,0.0718506673971812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,15,0.056661332647005715
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,31,0.056661332647005715
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,fp8,15,0.07202133536338806
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,fp8,31,0.0718506673971812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,63,0.05717333157857259
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,fp8,63,0.0718506673971812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,127,0.0727040022611618
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,fp8,127,0.08072533210118611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,fp8,1,0.1293653349081675
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,1,0.09966933727264404
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,255,0.24132267634073892
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,fp8,255,0.10342400272687276
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,3,0.09966933727264404
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,fp8,3,0.1293653349081675
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,7,0.09983999530474345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,fp8,7,0.12919466694196066
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,15,0.09983999530474345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,fp8,15,0.1293653349081675
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,31,0.09966933727264404
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,fp8,31,0.1293653349081675
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,63,0.10427733262379964
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,fp8,63,0.1293653349081675
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,127,0.22971733411153158
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,fp8,127,0.14813866217931113
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,1,0.1889280080795288
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,fp8,1,0.2481493353843689
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,3,0.18926932414372763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,255,0.4490240017573039
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,fp8,255,0.23961599667867026
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,fp8,3,0.2481493353843689
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,7,0.18926932414372763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,fp8,7,0.24797866741816202
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,15,0.18875734011332193
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,fp8,15,0.2481493353843689
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,31,0.19746132691701254
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,fp8,31,0.2481493353843689
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,63,0.2725546757380168
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,fp8,63,0.2481493353843689
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,127,0.4253013531366984
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,fp8,127,0.32392533620198566
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,1,0.3677866856257121
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,fp8,1,0.4858880043029785
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,3,0.3679573138554891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,255,0.839680035909017
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,fp8,255,0.4384426673253377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,fp8,3,0.48571733633677167
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,7,0.3679573138554891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,fp8,7,0.48622934023539227
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,15,0.3781973520914714
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,fp8,15,0.4858880043029785
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,fp8,31,0.49186134338378906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,31,0.4882773160934448
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,63,0.5137066841125488
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,fp8,63,0.5386240084966024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,127,0.7985493342081705
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,1,0.006826666494210561
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,3,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,1,0.021162666380405426
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,fp8,127,0.6224213441212972
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,3,0.021333334346612293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,255,1.6112640698750813
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,7,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,fp8,255,0.8280746936798096
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,7,0.021146667500336964
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,15,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,15,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,31,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,31,0.025600001215934753
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,63,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,127,0.008714666590094566
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,127,0.02595199892918269
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,255,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,63,0.025429333249727886
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,511,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,511,0.05256533126036326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,255,0.0314026673634847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,1023,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,2047,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,1023,0.09557333588600159
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,2047,0.18175999323527017
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,4095,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,4095,0.3543039957682292
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,8191,0.032085334261258446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,8191,0.6987199783325195
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,16383,0.04625066618124644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,16383,1.3878614107767742
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,32767,0.22272000710169473
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,1,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,1,0.021333334346612293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,32767,2.863274574279785
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,65535,0.43025068442026776
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,3,0.007173333317041397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,65535,5.777578353881836
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,3,0.021669333179791767
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,7,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,15,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,15,0.007653333246707916
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,31,0.025600001215934753
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,63,0.008346666892369589
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,63,0.025920001169045765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,127,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,127,0.025941332181294758
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,255,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,255,0.03134933362404505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,511,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,511,0.052906667192777
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,1023,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,1023,0.09557333588600159
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,2047,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,2047,0.1818986733754476
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,4095,0.03328000009059906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,4095,0.3543039957682292
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,8191,0.6993920008341471
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,8191,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,16383,0.22118399540583292
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,16383,1.3882026672363281
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,32767,0.4271786610285441
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,32767,2.9002933502197266
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,65535,0.8477013111114502
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,1,0.008010666817426682
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,3,0.008506666868925095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,3,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,65535,5.791402816772461
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,7,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,15,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,63,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,63,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,255,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,255,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,511,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,1023,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,511,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,1023,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,2047,0.034304000437259674
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,2047,0.04164266586303711
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,4095,0.056661332647005715
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,4095,0.07406933108965556
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,8191,0.2152106761932373
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,8191,0.14028799533843994
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,16383,0.42444801330566406
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,16383,0.27562665939331055
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,32767,0.845141331354777
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,32767,0.5374293327331543
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,65535,1.6856746673583984
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,1,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,65535,1.0618879795074463
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,3,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,3,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,7,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,15,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,31,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,63,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,255,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,255,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,511,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,511,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,1023,0.03549866626660029
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,1023,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,2047,0.04181333382924398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,2047,0.05529599885145823
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,4095,0.21384533246358237
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,4095,0.07492266595363617
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,8191,0.42137598991394043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,8191,0.2135039965311686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,16383,0.8378026485443115
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,16383,0.4097706476847331
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,32767,1.6747520764668782
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,32767,0.8060586452484131
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,1,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,1,0.019274666905403137
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,65535,1.5918080012003581
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,65535,3.357013384501139
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,3,0.006826666494210561
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,3,0.019306667149066925
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,7,0.007007999966541926
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,15,0.00730666642387708
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,15,0.019952000429232914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,7,0.019797333826621372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,31,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,63,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,31,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,63,0.024069334069887798
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,127,0.023573334018389385
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,255,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,511,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,511,0.030373332401116688
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,1023,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,1023,0.05167999863624573
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,2047,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,2047,0.09489599863688152
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,4095,0.025429333249727886
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,4095,0.18056533734003702
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,8191,0.03737599899371465
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,8191,0.3517013390858968
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,16383,0.05836800237496694
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,16383,0.6956373055775961
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,32767,0.20206934213638306
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,32767,1.434282620747884
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,1,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,65535,0.39082666238149005
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,1,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,65535,2.899968147277832
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,7,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,31,0.007285333548982938
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,15,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,63,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,127,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,255,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,127,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,511,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,255,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,511,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,1023,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,1023,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,2047,0.026965332527955372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,2047,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,4095,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,4095,0.040618665516376495
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,8191,0.0576853354771932
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,8191,0.07355733215808868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,16383,0.20138667027155557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,16383,0.1389226714769999
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,32767,0.38894931475321454
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,32767,0.2723840077718099
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,1,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,65535,0.5346986850102743
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,65535,0.769536018371582
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,7,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,15,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,31,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,63,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,63,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,255,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,511,0.01826133330663045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,511,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,1023,0.028501334289709728
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,2047,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,2047,0.024746666351954143
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,1023,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,4095,0.057002668579419456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,4095,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,8191,0.20138667027155557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,8191,0.07389866809050243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,16383,0.3927040100097656
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,16383,0.19353600343068442
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,32767,0.7797760168711344
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,32767,0.37597866853078205
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,1,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,65535,0.7391573588053385
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,3,0.007055999711155891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,65535,1.560746669769287
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,15,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,7,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,15,0.011781333635250727
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,31,0.00721066693464915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,63,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,31,0.011749333391586939
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,63,0.013440000514189402
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,127,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,127,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,255,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,255,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,511,0.008485333373149237
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,511,0.01826133330663045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,1023,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,2047,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,1023,0.020138667275508244
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,2047,0.019653332730134327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,4095,0.040448000033696495
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,4095,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,8191,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,8191,0.0194560003777345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,16383,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,16383,0.052906667192777
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,32767,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,32767,0.09506133198738098
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,65535,0.019626667102177937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,65535,0.17749333381652832
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,1,0.007162666569153468
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,1,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,131071,0.027818667391935985
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,3,0.006954666847983996
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,131071,0.33211733897527057
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,7,0.01163200040658315
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,15,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,15,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,31,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,31,0.011946666985750198
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,63,0.008181333541870117
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,63,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,127,0.014309333016475042
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,255,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,255,0.013872000078360239
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,511,0.008394666636983553
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,511,0.01894933357834816
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,1023,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,1023,0.019968000551064808
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,2047,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,2047,0.01911466692884763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,4095,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,4095,0.01826133330663045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,8191,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,8191,0.018432000031073887
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,16383,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,16383,0.052906667192777
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,32767,0.016384000579516094
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,32767,0.09489066402117412
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,65535,0.1776799956957499
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,65535,0.020138667275508244
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,131071,0.040106666584809623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,131071,0.33177600304285687
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,3,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,7,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,3,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,15,0.0124746672809124
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,31,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,63,0.008042666440208754
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,31,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,63,0.014314666390419006
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,127,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,255,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,255,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,511,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,511,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,1023,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,1023,0.01826133330663045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,2047,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,2047,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,4095,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,4095,0.018229333062966663
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,8191,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,8191,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,16383,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,16383,0.053632001082102455
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,32767,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,32767,0.09574400385220845
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,65535,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,65535,0.1786880095799764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,1,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,131071,0.05580799778302511
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,131071,0.3319466710090637
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,1,0.011616000284751257
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,7,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,7,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,15,0.007173333317041397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,15,0.012154666086037954
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,31,0.007482666522264481
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,31,0.012138667205969492
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,63,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,63,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,127,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,255,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,255,0.014378666877746582
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,511,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,511,0.017866666118303936
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,1023,0.01163200040658315
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,1023,0.019626667102177937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,2047,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,4095,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,2047,0.019285333653291065
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,4095,0.031727999448776245
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,8191,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,8191,0.05288533369700114
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,16383,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,16383,0.09538132945696513
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,32767,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,32767,0.17783466974894205
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,65535,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,65535,0.3299520015716553
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,131071,0.040448000033696495
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,131071,0.6353919903437296
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,3,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,7,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,7,0.011391999820868174
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,15,0.007663999994595845
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,15,0.011594666788975397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,31,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,31,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,63,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,63,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,127,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,255,0.00922133338948091
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,255,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,511,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,511,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,1023,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,1023,0.020138667275508244
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,2047,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,4095,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,2047,0.018453333526849747
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,4095,0.031231999397277832
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,8191,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,8191,0.053247998158137
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,16383,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,16383,0.09488000472386678
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,32767,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,32767,0.17802667617797852
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,65535,0.3298986752827962
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,65535,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,131071,0.05580799778302511
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,1,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,131071,0.6352213223775228
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,1,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,3,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,7,0.007280000175038974
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,7,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,15,0.007333333293596904
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,15,0.012618667135636011
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,31,0.007322666545708974
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,31,0.011946666985750198
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,63,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,63,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,127,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,255,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,255,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,511,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,511,0.018485333770513535
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,1023,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,1023,0.020138667275508244
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,2047,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,2047,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,4095,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,4095,0.032085334261258446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,8191,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,8191,0.053930665055910744
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,16383,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,16383,0.09608532985051473
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,32767,0.03345066557327906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,32767,0.17851734161376953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,65535,0.054101333022117615
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,65535,0.33075199524561566
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,1,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,131071,0.21077332894007364
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,3,0.007461333026488622
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,131071,0.6370986700057983
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,3,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,7,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,15,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,15,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,31,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,63,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,63,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,127,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,255,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,511,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,511,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,1023,0.01791999985774358
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,1023,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,2047,0.027136000494162243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,2047,0.024746666351954143
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,4095,0.04113066693147024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,4095,0.040789333482583366
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,8191,0.05751466751098633
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,8191,0.07389866809050243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,16383,0.20189867417017618
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,16383,0.13943466544151306
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,1,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,1,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,3,0.007461333026488622
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,32767,0.2727253238360087
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,32767,0.3928746779759725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,15,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,31,0.0074560002734263735
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,63,0.007333333293596904
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,255,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,255,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,511,0.018090666582187016
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,1023,0.02867199977238973
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,511,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,1023,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,2047,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,2047,0.024746666351954143
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,4095,0.05717333157857259
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,4095,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,8191,0.20121600230534872
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,8191,0.07406933108965556
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,16383,0.39048532644907635
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,16383,0.19438934326171875
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,1,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,32767,0.3800746599833171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,32767,0.7729492982228597
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,3,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,3,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,7,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,15,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,15,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,31,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,31,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,63,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,63,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,127,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,127,0.043178667624791466
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,255,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,255,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,511,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,511,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,1023,0.03908266623814901
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,1023,0.025770666698614757
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,2047,0.056832000613212585
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,2047,0.04351999859015147
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,4095,0.20138667027155557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,4095,0.07611733178297679
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,8191,0.3938986857732137
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,8191,0.19353600343068442
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,16383,0.7811413606007894
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,16383,0.3737599849700928
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,1,0.011061333119869232
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,32767,0.7488853136698405
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,32767,1.556821346282959
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,3,0.007280000175038974
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,7,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,7,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,15,0.007333333293596904
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,15,0.01139733319481214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,31,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,63,0.007882666463653246
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,31,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,63,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,127,0.009226666763424873
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,255,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,127,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,255,0.013701333353916803
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,511,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,511,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,1023,0.012069333344697952
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,1023,0.019285333653291065
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,2047,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,2047,0.0339626669883728
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,4095,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,4095,0.05397866666316986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,8191,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,8191,0.09148800373077393
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,16383,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,16383,0.15752533078193665
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,32767,0.025770666698614757
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,32767,0.2927146752675374
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,65535,0.03601066768169403
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,65535,0.5763413508733114
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,1,0.007045333584149678
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,1,0.010928000013033548
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,131071,0.05649066468079885
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,131071,0.7203839619954427
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,3,0.0074879998962084455
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,7,0.011050666371981302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,15,0.007055999711155891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,15,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,31,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,31,0.012794667234023413
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,63,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,63,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,127,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,255,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,255,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,511,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,511,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,1023,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,1023,0.019968000551064808
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,2047,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,2047,0.032255999743938446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,4095,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,4095,0.054101333022117615
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,8191,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,8191,0.09181867043177287
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,16383,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,16383,0.15752533078193665
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,32767,0.034815999368826546
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,32767,0.29286400477091473
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,65535,0.054272000988324486
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,65535,0.5766826470692953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,131071,0.20855466524759927
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,131071,0.7224319775899252
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,3,0.007189333438873291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,3,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,7,0.007162666569153468
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,15,0.007311999797821045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,7,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,15,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,31,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,31,0.011946666985750198
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,63,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,63,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,127,0.008506666868925095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,255,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,255,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,511,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,511,0.01911466692884763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,1023,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,1023,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,2047,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,2047,0.03242666771014532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,4095,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,4095,0.053930665055910744
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,8191,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,8191,0.09181867043177287
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,16383,0.03601066768169403
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,16383,0.15752533078193665
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,32767,0.05495466788609823
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,32767,0.2930346727371216
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,65535,0.2070186734199524
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,65535,0.5771946509679159
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,1,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,131071,0.40294400850931805
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,131071,0.7425706386566162
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,3,0.0075040000180403394
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,15,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,31,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,63,0.007637333124876022
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,127,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,255,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,511,0.01826133330663045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,511,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,1023,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,1023,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,2047,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,2047,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,4095,0.05734399954477946
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,4095,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,8191,0.2039466698964437
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,1,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,8191,0.07406933108965556
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,1,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,16383,0.39372801780700684
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,16383,0.1971199909845988
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,3,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,3,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,7,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,7,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,15,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,31,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,31,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,63,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,63,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,127,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,127,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,255,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,255,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,511,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,511,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,1023,0.03908266623814901
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,1023,0.025770666698614757
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,2047,0.05751466751098633
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,2047,0.04351999859015147
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,4095,0.20104533433914185
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,4095,0.0766293356815974
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,8191,0.3916800022125244
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,8191,0.19387733936309814
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,1,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,1,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,3,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,16383,0.37444265683492023
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,16383,0.7761920293172201
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,3,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,7,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,7,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,15,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,15,0.016895999511082966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,31,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,31,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,63,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,63,0.017237332959969837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,127,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,255,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,127,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,255,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,511,0.040106666584809623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,511,0.032085334261258446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,1023,0.056832000613212585
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,1023,0.04642133414745331
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,2047,0.2027519941329956
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,2047,0.08089600006739299
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,4095,0.3944106499354045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,4095,0.1955839991569519
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,8191,0.7801173528035482
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,8191,0.37614933649698895
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,1,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,1,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,3,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,16383,1.5604052543640137
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,16383,0.7374506791432699
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,3,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,7,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,7,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,15,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,31,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,31,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,63,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,63,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,127,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,127,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,255,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,255,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,511,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,511,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,1023,0.03908266623814901
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,1023,0.025770666698614757
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,2047,0.05751466751098633
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,2047,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,1,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,4095,0.2027519941329956
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,1,0.017237332959969837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,4095,0.0769706666469574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,3,0.016895999511082966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,3,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,7,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,7,0.017407999684413273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,15,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,15,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,31,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,31,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,63,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,63,0.017407999684413273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,127,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,127,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,255,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,255,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,511,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,511,0.03089066594839096
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,1023,0.05717333157857259
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,1023,0.04710400104522705
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,2047,0.20309333006540933
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,2047,0.08072533210118611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,1,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,1,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,4095,0.39560532569885254
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,3,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,3,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,4095,0.19643733898798624
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,7,0.027818667391935985
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,7,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,15,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,15,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,31,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,31,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,63,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,63,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,127,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,127,0.03515733281771342
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,255,0.03703466554482778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,255,0.04232533276081085
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,511,0.053247998158137
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,511,0.05717333157857259
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,1023,0.08635733524958293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,1023,0.20070399840672812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,2047,0.39867734909057617
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,2047,0.1986560026804606
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,1,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,1,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,4095,0.3838293155034383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,4095,0.7866026560465494
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,3,0.016384000579516094
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,3,0.017237332959969837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,7,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,7,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,15,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,15,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,31,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,31,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,63,0.016384000579516094
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,63,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,127,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,127,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,255,0.030378667016824085
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,255,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,511,0.040789333482583366
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,511,0.032085334261258446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,1023,0.05751466751098633
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,1023,0.04710400104522705
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,2047,0.20548266172409058
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,1,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,1,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,3,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,3,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,2047,0.08106666803359985
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,7,0.02935466667016347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,7,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,15,0.028160000840822857
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,15,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,31,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,31,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,63,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,63,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,127,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,127,0.03566933423280716
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,255,0.03703466554482778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,255,0.04266666869322459
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,511,0.053247998158137
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,511,0.056661332647005715
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,1023,0.20121600230534872
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,1023,0.08686932921409607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,1,0.05034666756788889
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,2047,0.3974826733271281
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,2047,0.19899733861287436
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,1,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,3,0.05034666756788889
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,3,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,7,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,7,0.05034666756788889
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,15,0.05034666756788889
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,15,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,31,0.05034666756788889
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,31,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,63,0.05000533163547516
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,63,0.05034666756788889
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,127,0.05034666756788889
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,127,0.06382933259010315
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,255,0.06570666531721751
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,255,0.07816533247629802
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,511,0.19780266284942627
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,511,0.10478933652242024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,1023,0.39133866628011066
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,1023,0.19387733936309814
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,2047,0.3787093162536621
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,3,0.011205332974592844
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,2047,0.7773866653442383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,7,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,31,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,31,0.011946666985750198
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,63,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,127,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,255,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,255,0.014485333114862442
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,511,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,511,0.019621333728233974
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,1023,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,1023,0.0341333324710528
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,2047,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,2047,0.048810665806134544
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,4095,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,4095,0.07577600081761678
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,8191,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,8191,0.13567999998728433
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,16383,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,16383,0.24968532721201578
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,32767,0.034304000437259674
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,32767,0.4840693473815918
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,65535,0.05461333195368449
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,65535,0.7204053401947021
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,1,0.01108266661564509
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,3,0.00702400008837382
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,3,0.010965333630641302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,131071,0.20957867304484049
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,131071,1.4344533284505208
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,7,0.007280000175038974
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,7,0.011285333583752314
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,15,0.007327999919652939
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,31,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,31,0.011946666985750198
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,63,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,63,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,127,0.008341333518425623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,255,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,255,0.015034666905800501
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,511,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,511,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,1023,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,1023,0.03259733319282532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,2047,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,2047,0.04898133377234141
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,4095,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,4095,0.07543466488520305
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,8191,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,8191,0.13570132851600647
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,16383,0.03566933423280716
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,16383,0.25036799907684326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,32767,0.05478399991989136
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,32767,0.4853760004043579
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,65535,0.20462934176127115
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,65535,0.7191893259684244
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,1,0.007194666812817256
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,131071,0.40004265308380127
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,3,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,131071,1.4612479209899902
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,7,0.011061333119869232
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,15,0.00730666642387708
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,15,0.011802667131026586
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,31,0.007663999994595845
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,31,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,63,0.008357333640257517
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,63,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,127,0.008506666868925095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,127,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,511,0.03857066730658213
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,255,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,511,0.01791999985774358
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,1023,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,1023,0.03259733319282532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,2047,0.04863999783992767
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,2047,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,4095,0.025770666698614757
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,4095,0.0759200006723404
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,8191,0.03754666695992152
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,8191,0.1360213359196981
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,16383,0.058880001306533813
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,16383,0.2500266631444295
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,32767,0.20206934213638306
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,32767,0.48469332853953045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,65535,0.3949226538340251
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,65535,0.739840030670166
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,1,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,1,0.029696000119050343
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,131071,0.7804586887359619
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,131071,1.4714879989624023
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,3,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,3,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,7,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,7,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,15,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,15,0.02935466667016347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,31,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,31,0.029696000119050343
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,63,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,63,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,127,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,127,0.0363520011305809
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,255,0.03703466554482778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,255,0.043178667624791466
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,511,0.05273599922657013
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,511,0.05836800237496694
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,1,0.05034666756788889
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,1,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,3,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,3,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,1023,0.08789333701133728
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,1023,0.2063360015551249
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,7,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,7,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,15,0.05034666756788889
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,15,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,31,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,31,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,63,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,63,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,127,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,127,0.06468266745408376
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,255,0.06604800124963124
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,255,0.07918933530648549
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,511,0.20121600230534872
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,1,0.09352533022562663
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,511,0.10461866855621338
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,1,0.09540266791979472
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,3,0.09335466225941975
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,1023,0.3942399819691976
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,1023,0.19797333081563315
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,3,0.09540266791979472
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,7,0.09352533022562663
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,7,0.09540266791979472
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,15,0.09335466225941975
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,15,0.09540266791979472
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,31,0.09335466225941975
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,31,0.09557333588600159
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,63,0.09352533022562663
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,63,0.09540266791979472
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,127,0.09437867005666097
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,127,0.1220266620318095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,255,0.1971199909845988
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,255,0.1508693297704061
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,511,0.38860801855723065
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,511,0.2167466680208842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,1023,0.37444265683492023
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,1,0.051541333397229515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,1,0.05239466826121012
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,1023,0.7717546621958414
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,3,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,3,0.05239466826121012
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,7,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,7,0.05256533126036326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,15,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,15,0.05239466826121012
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,31,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,31,0.05239466826121012
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,63,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,63,0.05239466826121012
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,127,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,127,0.06621866424878438
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,1,0.09642666578292847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,1,0.09454933802286784
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,255,0.06690133114655812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,255,0.08021333316961925
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,3,0.09437867005666097
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,3,0.09642666578292847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,511,0.20718934138615927
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,511,0.10769066214561462
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,7,0.09454933802286784
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,7,0.09642666578292847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,15,0.09454933802286784
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,15,0.09659733374913533
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,31,0.09454933802286784
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,31,0.09642666578292847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,63,0.09454933802286784
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,63,0.09642666578292847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,127,0.09506133198738098
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,127,0.1230506698290507
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,255,0.20428800582885742
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,255,0.1508693297704061
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,1,0.18363734086354574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,3,0.18363734086354574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,1,0.18551466862360635
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,3,0.1853440006573995
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,511,0.4016000032424927
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,7,0.18363734086354574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,7,0.18568533658981323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,511,0.21657600005467734
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,15,0.18363734086354574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,15,0.18551466862360635
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,31,0.1838080088297526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,31,0.18551466862360635
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,63,0.1838080088297526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,63,0.18551466862360635
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,127,0.2505386670430501
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,127,0.2373973329861959
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,1,0.007391999786098798
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,255,0.4015786647796631
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,1,0.010879999647537867
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,3,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,255,0.3037866751352946
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,7,0.0074986666440963745
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,7,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,15,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,31,0.007514666765928268
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,31,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,63,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,511,0.43059198061625165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,511,0.7664639949798584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,63,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,127,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,255,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,255,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,511,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,511,0.0314026673634847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,1023,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,1023,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,2047,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,2047,0.06570666531721751
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,4095,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,4095,0.10922666390736897
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,8191,0.024746666351954143
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,8191,0.19592533508936563
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,16383,0.03583999971548716
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,16383,0.37034666538238525
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,32767,0.05495466788609823
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,32767,0.7202186584472656
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,65535,0.2044586737950643
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,65535,1.4310399691263835
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,1,0.007301333049933116
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,1,0.010949333508809408
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,131071,0.4007253249486287
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,3,0.007322666545708974
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,131071,2.902186711629232
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,7,0.007477333148320516
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,15,0.011413333316644033
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,31,0.007802666475375493
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,31,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,63,0.008538666491707167
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,63,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,127,0.008714666590094566
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,127,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,255,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,511,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,511,0.031231999397277832
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,1023,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,1023,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,2047,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,2047,0.06553600231806438
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,4095,0.025600001215934753
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,4095,0.10871466994285583
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,8191,0.03737599899371465
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,8191,0.1960960030555725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,16383,0.0580266664425532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,16383,0.3701759974161784
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,32767,0.20155733823776245
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,32767,0.7193813323974609
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,65535,0.389631986618042
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,1,0.007663999994595845
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,65535,1.4580052693684895
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,131071,0.7693653106689453
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,1,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,131071,2.9093545277913413
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,3,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,7,0.0074506668994824094
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,7,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,15,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,31,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,15,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,63,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,63,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,127,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,127,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,255,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,511,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,511,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,1023,0.01791999985774358
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,1023,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,2047,0.026965332527955372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,2047,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,4095,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,4095,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,8191,0.05751466751098633
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,8191,0.0730453332265218
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,16383,0.20206934213638306
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,16383,0.1384106675783793
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,32767,0.39372801780700684
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,32767,0.2720426718393962
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,65535,0.777728001276652
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,65535,0.5341866811116537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,1,0.007333333293596904
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,1,0.021503999829292297
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,3,0.02170666555563609
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,3,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,131071,1.5453866322835286
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,131071,1.061205307642619
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,7,0.007151999821265538
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,15,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,7,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,15,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,31,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,31,0.025600001215934753
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,63,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,63,0.025994665920734406
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,127,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,127,0.026399999856948853
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,255,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,255,0.03141866624355316
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,511,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,511,0.053077335158983864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,1023,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,1023,0.09593600034713745
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,2047,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,2047,0.18174399932225546
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,4095,0.03379199902216593
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,4095,0.3548159996668498
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,8191,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,8191,0.6990506649017334
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,16383,0.2198186715443929
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,16383,1.385983943939209
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,32767,0.4227413336435954
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,1,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,32767,2.898773193359375
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,3,0.007807999849319458
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,65535,0.8374613126118978
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,65535,5.787647883097331
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,7,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,7,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,127,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,63,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,255,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,255,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,511,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,511,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,1023,0.023039999107519787
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,1023,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,2047,0.034815999368826546
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,2047,0.04215466479460398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,4095,0.05597866574923197
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,4095,0.07492266595363617
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,8191,0.21401600042978922
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,8191,0.14045866330464682
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,16383,0.4179626703262329
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,16383,0.2759679953257243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,32767,0.8326826890309652
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,32767,0.5389653444290161
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,65535,1.6599040031433105
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,65535,1.06222931543986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,3,0.008320000022649765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,7,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,15,0.009530666594703993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,31,0.00847999999920527
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,63,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,127,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,255,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,255,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,511,0.02218666672706604
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,511,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,1023,0.03566933423280716
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,1023,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,2047,0.055125330885251365
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,2047,0.04215466479460398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,4095,0.21401600042978922
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,4095,0.07509333391984303
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,8191,0.21504000822703043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,8191,0.42188799381256104
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,16383,0.8352426687876383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,16383,0.4118186632792155
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,32767,0.8023040294647217
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,32767,1.6665600140889485
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,1,0.007311999797821045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,1,0.01302933320403099
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,65535,3.3421653111775718
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,3,0.007301333049933116
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,3,0.013295999417702356
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,65535,1.586858590443929
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,7,0.007296000296870868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,7,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,15,0.007647999872763951
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,31,0.01481066644191742
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,15,0.013370666652917862
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,31,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,127,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,127,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,255,0.008357333640257517
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,255,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,511,0.02067199970285098
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,1023,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,1023,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,2047,0.012624000509579977
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,511,0.011749333391586939
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,2047,0.02046400060256322
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,4095,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,4095,0.020138667275508244
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,8191,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,8191,0.053957333167394005
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,16383,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,16383,0.09676800171534221
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,32767,0.1812480092048645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,32767,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,65535,0.024746666351954143
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,65535,0.33672531445821124
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,1,0.007205333560705185
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,1,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,131071,0.03566933423280716
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,3,0.007327999919652939
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,131071,0.646997332572937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,3,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,15,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,7,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,31,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,31,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,63,0.008863999818762144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,127,0.008218666538596153
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,127,0.015184000134468079
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,255,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,255,0.02051199972629547
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,511,0.011760000139474869
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,511,0.021333334346612293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,1023,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,1023,0.021162666380405426
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,2047,0.013151999562978745
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,2047,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,4095,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,4095,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,8191,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,8191,0.054272000988324486
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,16383,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,16383,0.09659733374913533
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,32767,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,32767,0.18090667327245077
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,65535,0.027818667391935985
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,65535,0.33672531445821124
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,1,0.007157333195209503
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,1,0.014373333503802618
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,131071,0.051882664362589516
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,131071,0.6478506724039713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,3,0.014368000129858652
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,7,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,15,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,15,0.014826666563749313
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,31,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,31,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,63,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,127,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,63,0.014970666418472925
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,127,0.01581866666674614
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,255,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,255,0.0204373337328434
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,511,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,511,0.021333334346612293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,1023,0.012298667182525
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,1023,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,2047,0.011946666985750198
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,2047,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,4095,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,4095,0.020442667106787365
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,8191,0.016042667130629223
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,8191,0.05563733478387197
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,16383,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,16383,0.09864532947540283
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,32767,0.1824693282445272
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,32767,0.032085334261258446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,65535,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,65535,0.33723731835683185
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,131071,0.23296000560124716
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,1,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,131071,0.6488746802012125
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,3,0.007311999797821045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,3,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,7,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,7,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,15,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,31,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,31,0.014864000181357065
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,63,0.008389333263039589
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,127,0.008357333640257517
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,127,0.015541333705186844
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,63,0.01498666654030482
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,255,0.008410666758815447
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,511,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,255,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,511,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,1023,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,1023,0.021162666380405426
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,2047,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,2047,0.032085334261258446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,4095,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,4095,0.054272000988324486
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,8191,0.01621333385507266
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,8191,0.09642666578292847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,16383,0.017237332959969837
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,16383,0.1812480092048645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,32767,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,32767,0.33587201436360675
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,65535,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,65535,0.6452906529108683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,131071,0.05444266895453135
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,1,0.007125333572427432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,131071,1.2690773010253906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,1,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,3,0.007226666435599327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,3,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,7,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,15,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,31,0.016042667130629223
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,31,0.008000000069538752
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,63,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,127,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,127,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,255,0.019968000551064808
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,511,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,511,0.020992000897725422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,1023,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,1023,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,2047,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,2047,0.032144000132878624
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,4095,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,4095,0.05422399938106537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,8191,0.01621333385507266
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,8191,0.09676800171534221
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,16383,0.01826133330663045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,16383,0.18110400438308716
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,32767,0.031744000812371574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,32767,0.33604268232981366
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,65535,0.04369066655635834
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,1,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,65535,0.6449493169784546
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,1,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,131071,0.22971733411153158
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,131071,1.2692480087280273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,7,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,3,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,7,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,15,0.007642666498819987
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,15,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,31,0.007690666864315669
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,31,0.01684800038735072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,63,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,63,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,127,0.008709333216150602
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,127,0.015872000406185787
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,255,0.020293333878119785
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,511,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,511,0.021162666380405426
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,1023,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,1023,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,2047,0.03364266703526179
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,2047,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,4095,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,4095,0.05526400109132131
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,8191,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,8191,0.0981333355108897
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,16383,0.027818667391935985
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,16383,0.18277867635091147
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,32767,0.04471466441949209
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,32767,0.33689598242441815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,65535,0.22920533021291098
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,65535,0.6459733247756958
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,1,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,131071,0.4452693462371826
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,131071,1.3056000073750813
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,3,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,3,0.009343999748428663
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,15,0.008527999743819237
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,31,0.008330666770537695
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,31,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,63,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,255,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,255,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,511,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,511,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,1023,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,1023,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,2047,0.03498666733503342
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,2047,0.04232533276081085
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,4095,0.0576853354771932
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,4095,0.07509333391984303
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,8191,0.21640533208847046
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,8191,0.14062933127085367
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,16383,0.42239999771118164
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,16383,0.27665066719055176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,32767,0.8376320203145345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,32767,0.5389653444290161
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,3,0.008496000121037165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,7,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,7,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,31,0.008517333616813024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,63,0.008512000242869059
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,255,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,255,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,511,0.022357332209746044
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,511,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,1023,0.03601066768169403
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,1023,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,2047,0.05495466788609823
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,2047,0.04232533276081085
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,4095,0.07543466488520305
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,4095,0.21452800432840982
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,8191,0.21538132429122925
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,8191,0.4189866781234741
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,16383,0.8301226298014323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,16383,0.41625599066416424
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,1,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,1,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,3,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,32767,0.8212479750315348
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,32767,1.6491519610087078
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,7,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,3,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,7,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,15,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,15,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,31,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,31,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,63,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,127,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,127,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,63,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,255,0.02218666672706604
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,255,0.017407999684413273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,511,0.0363520011305809
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,511,0.02679466704527537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,1023,0.05273599922657013
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,1023,0.04334933559099833
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,2047,0.21316266059875488
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,2047,0.07816533247629802
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,4095,0.4184746742248535
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,4095,0.21708800395329794
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,8191,0.8277333577473959
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,8191,0.41813333829243976
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,16383,0.8183466593424479
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,16383,1.6585386594136555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,1,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,32767,3.319295883178711
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,3,0.013114667187134424
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,32767,1.6030720074971516
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,7,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,15,0.0075040000180403394
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,31,0.007989333321650824
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,15,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,31,0.015311999867359797
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,127,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,255,0.019797333826621372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,511,0.01180800050497055
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,511,0.02059200033545494
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,1023,0.012954667210578918
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,1023,0.033957332372665405
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,2047,0.05563733478387197
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,4095,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,2047,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,4095,0.09318400422732036
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,8191,0.017407999684413273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,8191,0.16110933820406595
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,16383,0.019285333653291065
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,16383,0.2993493278821309
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,32767,0.03293866664171219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,32767,0.5858986775080363
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,65535,0.045567999283472695
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,65535,1.153167963027954
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,1,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,131071,0.2290346622467041
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,1,0.007333333293596904
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,131071,1.4382559458414714
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,3,0.01312000056107839
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,7,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,7,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,15,0.007311999797821045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,15,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,31,0.008047999814152718
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,63,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,31,0.01471466695268949
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,127,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,127,0.009205333267649015
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,255,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,255,0.019626667102177937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,511,0.020992000897725422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,511,0.011834666132926941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,1023,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,1023,0.0339626669883728
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,2047,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,2047,0.05596800148487091
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,4095,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,8191,0.023210667073726654
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,4095,0.09368000427881877
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,16383,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,8191,0.16076800227165222
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,16383,0.2998613317807515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,32767,0.04437333345413208
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,32767,0.5869119962056478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,65535,0.22715733448664346
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,1,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,1,0.007354666789372762
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,65535,1.1535360018412273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,3,0.007391999786098798
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,131071,0.4410026470820109
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,131071,1.4619305928548176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,7,0.013157332936922709
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,3,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,15,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,31,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,31,0.014661333213249842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,127,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,127,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,255,0.0194560003777345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,511,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,511,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,1023,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,1023,0.0339626669883728
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,2047,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,2047,0.05580799778302511
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,4095,0.09403733412424724
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,4095,0.020992000897725422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,8191,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,8191,0.1609386702378591
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,16383,0.04625066618124644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,16383,0.2996906638145447
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,32767,0.22169599930445352
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,32767,0.5865813493728638
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,65535,0.43622398376464844
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,65535,1.18340269724528
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,131071,0.8690346876780192
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,131071,1.4854826927185059
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,3,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,7,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,15,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,7,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,31,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,63,0.008352000266313553
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,255,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,255,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,511,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,511,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,1023,0.03618133316437403
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,1023,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,2047,0.0554666668176651
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,2047,0.04232533276081085
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,4095,0.21896533171335855
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,4095,0.07509333391984303
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,8191,0.42239999771118164
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,8191,0.21538132429122925
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,1,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,1,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,3,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,16383,0.8349013328552246
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,16383,0.4123306671778361
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,3,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,7,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,7,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,15,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,15,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,31,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,31,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,63,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,63,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,127,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,127,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,255,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,255,0.02218666672706604
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,511,0.03618133316437403
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,511,0.02679466704527537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,1023,0.043178667624791466
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,1023,0.05256533126036326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,2047,0.21589332818984985
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,2047,0.07816533247629802
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,4095,0.42188799381256104
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,4095,0.21845332781473795
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,8191,0.8321706453959147
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,8191,0.4174506664276123
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,1,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,1,0.019968000551064808
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,3,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,3,0.019968000551064808
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,16383,1.6518826484680176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,16383,0.8156159718831381
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,7,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,15,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,7,0.019797333826621372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,15,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,31,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,31,0.020138667275508244
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,63,0.017407999684413273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,63,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,127,0.021162666380405426
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,127,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,255,0.036864000062147774
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,255,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,511,0.05376000205675761
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,511,0.049322664737701416
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,1023,0.21230934063593546
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,1023,0.0820906658967336
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,2047,0.4220586617787679
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,2047,0.22050132354100546
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,4095,0.4102826515833537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,4095,0.8330240249633789
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,8191,1.664170742034912
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,8191,0.8014506498972574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,1,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,1,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,3,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,3,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,16383,3.3281707763671875
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,16383,1.5842986106872559
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,7,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,7,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,15,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,15,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,31,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,31,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,63,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,63,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,127,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,255,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,255,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,511,0.036864000062147774
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,511,0.02679466704527537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,1023,0.053247998158137
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,1023,0.043178667624791466
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,2047,0.233130673567454
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,2047,0.07867733140786488
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,1,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,1,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,4095,0.42496001720428467
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,3,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,4095,0.22220800320307413
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,3,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,7,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,7,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,15,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,15,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,31,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,31,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,63,0.0170666662355264
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,63,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,127,0.021503999829292297
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,127,0.023039999107519787
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,255,0.03703466554482778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,511,0.053930665055910744
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,255,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,511,0.04949333270390829
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,1023,0.0820906658967336
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,1023,0.2135039965311686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,2047,0.4261546532313029
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,2047,0.2198186715443929
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,1,0.03618133316437403
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,1,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,3,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,3,0.03669333209594091
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,4095,0.41096532344818115
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,4095,0.8364373048146566
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,7,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,7,0.0365226666132609
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,15,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,15,0.0365226666132609
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,31,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,31,0.054272000988324486
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,63,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,63,0.03618133316437403
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,127,0.03703466554482778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,127,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,255,0.04693333307902018
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,255,0.053247998158137
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,511,0.21435733636220297
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,511,0.09267200032869975
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,1023,0.4275199969609578
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,1023,0.22323199113210043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,2047,0.8477013111114502
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,2047,0.4254719813664754
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,1,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,4095,1.6837973594665527
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,4095,0.8275626500447592
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,1,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,3,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,3,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,7,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,7,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,15,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,15,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,31,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,63,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,31,0.020992000897725422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,63,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,127,0.020992000897725422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,127,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,255,0.037205333511034645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,255,0.029696000119050343
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,511,0.04949333270390829
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,511,0.05444266895453135
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,1023,0.08226133386294048
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,1023,0.2187946637471517
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,2047,0.4293973445892334
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,1,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,1,0.037205333511034645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,2047,0.22732800245285034
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,3,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,3,0.037205333511034645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,7,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,7,0.037205333511034645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,15,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,15,0.03703466554482778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,31,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,63,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,31,0.037205333511034645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,63,0.037205333511034645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,127,0.03703466554482778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,127,0.04215466479460398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,255,0.04915200173854828
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,255,0.05358933409055074
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,511,0.21760000785191855
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,511,0.09250133236249287
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,1023,0.43246932824452716
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,1023,0.2254506746927897
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,2047,0.8581120173136393
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,1,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,1,0.06570666531721751
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,2047,0.4251306851704915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,3,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,3,0.06587733328342438
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,7,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,7,0.06587733328342438
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,15,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,15,0.06587733328342438
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,31,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,31,0.06587733328342438
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,63,0.05085866649945577
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,63,0.06587733328342438
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,127,0.06621866424878438
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,127,0.07543466488520305
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,255,0.21026132504145303
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,255,0.09659733374913533
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,511,0.41779200236002606
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,511,0.211626668771108
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,1023,0.8383146921793619
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,1023,0.41813333829243976
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,1,0.007333333293596904
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,1,0.012645332763592402
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,3,0.00730666642387708
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,3,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,2047,1.6720213890075684
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,2047,0.8065706888834635
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,7,0.01231466606259346
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,15,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,31,0.008010666817426682
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,31,0.01421333352724711
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,63,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,63,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,127,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,255,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,255,0.01911466692884763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,511,0.0124746672809124
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,511,0.0341333324710528
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,1023,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,2047,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,1023,0.05085866649945577
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,2047,0.07816533247629802
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,4095,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,4095,0.13960533340771994
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,8191,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,8191,0.25570666790008545
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,16383,0.02867199977238973
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,16383,0.4930560191472371
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,32767,0.045738667249679565
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,32767,0.9612159729003906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,65535,0.2230613430341085
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,65535,1.435306708017985
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,1,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,131071,0.4336640040079753
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,3,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,131071,2.9050881067911782
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,7,0.013349333157142004
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,15,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,31,0.00789866658548514
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,31,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,63,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,127,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,255,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,255,0.0194560003777345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,511,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,511,0.034645333886146545
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,1023,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,1023,0.05034666756788889
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,2047,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,2047,0.07850666840871175
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,4095,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,4095,0.13943466544151306
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,8191,0.0315733328461647
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,8191,0.2563413381576538
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,16383,0.045226668318112694
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,16383,0.4925440152486165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,32767,0.22101332743962607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,32767,0.9620479742685953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,65535,0.4495360056559245
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,1,0.007327999919652939
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,65535,1.4590293566385906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,1,0.013280000537633896
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,131071,0.8622079690297445
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,3,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,131071,2.9177172978719077
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,7,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,15,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,31,0.0081386665503184
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,31,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,63,0.008383999889095625
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,127,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,255,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,255,0.0194560003777345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,511,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,511,0.03433600068092346
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,1023,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,1023,0.05085866649945577
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,2047,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,2047,0.07850666840871175
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,4095,0.03293866664171219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,4095,0.13924266894658408
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,8191,0.049322664737701416
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,8191,0.25600000222524005
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,16383,0.2193066676457723
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,16383,0.49373865127563477
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,32767,0.4270079930623372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,32767,0.9854293664296468
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,65535,0.8524800141652426
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,65535,1.4837759335835774
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,131071,1.6991573969523113
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,1,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,131071,2.9533866246541343
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,1,0.03822933385769526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,3,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,3,0.038058665891488395
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,7,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,7,0.03822933385769526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,15,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,15,0.03822933385769526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,31,0.03822933385769526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,31,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,63,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,63,0.038058665891488395
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,127,0.03822933385769526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,127,0.042837331692377724
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,255,0.05273599922657013
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,255,0.05461333195368449
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,1,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,511,0.09335466225941975
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,511,0.2239146629969279
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,1,0.06690133114655812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,1023,0.4410026470820109
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,1023,0.23125332593917847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,3,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,3,0.06690133114655812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,7,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,7,0.06673066814740498
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,15,0.051541333397229515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,15,0.06673066814740498
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,31,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,63,0.051541333397229515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,31,0.06673066814740498
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,63,0.06690133114655812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,127,0.06690133114655812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,127,0.07611733178297679
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,255,0.21486934026082358
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,255,0.09779199957847595
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,511,0.4266666571299235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,511,0.21589332818984985
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,1,0.09523199995358785
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,1,0.12424533565839131
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,3,0.09506133198738098
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,1023,0.8485546906789144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,3,0.12424533565839131
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,1023,0.4210346539815267
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,7,0.09489066402117412
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,7,0.12424533565839131
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,15,0.09471999605496724
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,15,0.12424533565839131
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,31,0.09489066402117412
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,63,0.09608532985051473
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,31,0.12424533565839131
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,63,0.12424533565839131
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,127,0.20121600230534872
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,127,0.14455466469128928
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,255,0.41147732734680176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,255,0.21179733673731485
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,511,0.8219306468963623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,511,0.4049919843673706
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,1023,0.8033280372619629
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,1023,1.6581974029541016
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,1,0.05341866612434387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,1,0.06894933183987935
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,3,0.05341866612434387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,3,0.06877866884072621
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,7,0.05358933409055074
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,7,0.06929066777229309
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,15,0.05341866612434387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,31,0.05341866612434387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,15,0.06911999980608623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,31,0.06877866884072621
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,63,0.05341866612434387
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,63,0.06877866884072621
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,127,0.06894933183987935
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,127,0.07816533247629802
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,1,0.09642666578292847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,255,0.2254506746927897
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,255,0.10018133123715718
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,1,0.1262933313846588
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,511,0.2285226583480835
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,511,0.4396373430887858
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,3,0.09659733374913533
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,3,0.1262933313846588
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,7,0.09642666578292847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,7,0.1262933313846588
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,15,0.09642666578292847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,15,0.1262933313846588
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,31,0.09659733374913533
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,31,0.1262933313846588
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,63,0.09762133161226909
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,63,0.1262933313846588
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,127,0.20992000897725424
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,127,0.14660267035166422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,255,0.4251306851704915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,255,0.22203733523686728
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,1,0.18568533658981323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,1,0.2450773318608602
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,3,0.18568533658981323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,3,0.24524799982706705
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,7,0.18568533658981323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,511,0.8401920000712076
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,7,0.24524799982706705
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,511,0.4184746742248535
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,15,0.18568533658981323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,15,0.2450773318608602
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,31,0.18705066045125326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,31,0.24541866779327393
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,63,0.2728959918022156
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,63,0.24524799982706705
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,127,0.4041386842727661
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,127,0.311296006043752
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,1,0.0074453335255384445
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,1,0.03908266623814901
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,255,0.8096426328023275
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,3,0.01246400053302447
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,255,0.41523198286692303
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,7,0.007567999884486198
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,7,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,15,0.007706666365265846
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,511,1.6283307075500488
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,15,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,31,0.008346666892369589
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,31,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,511,0.7877973715464274
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,63,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,63,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,127,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,255,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,255,0.03310933212439219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,511,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,1023,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,511,0.045567999283472695
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,1023,0.06739200154940288
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,2047,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,2047,0.11246933539708455
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,4095,0.022015998760859173
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,4095,0.20087466637293497
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,8191,0.032085334261258446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,8191,0.37751468022664386
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,16383,0.04607999821503957
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,16383,0.7307946681976318
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,32767,0.2193066676457723
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,32767,1.4351946512858074
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,65535,0.42581331729888916
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,65535,2.897893269856771
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,131071,0.8429226875305176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,1,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,3,0.007663999994595845
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,3,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,131071,5.79481569925944
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,7,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,7,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,15,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,31,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,31,0.015856000284353893
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,63,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,127,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,127,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,63,0.01498666654030482
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,255,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,255,0.03293866664171219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,511,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,511,0.045909335215886436
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,1023,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,1023,0.06775466601053874
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,2047,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,2047,0.11131200194358826
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,4095,0.03310933212439219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,4095,0.20138667027155557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,8191,0.04949333270390829
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,8191,0.37887998421986896
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,16383,0.21811199188232422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,16383,0.7311360041300455
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,32767,0.423253337542216
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,32767,1.457322597503662
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,65535,0.8424106438954672
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,1,0.008005333443482717
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,65535,2.9078187942504883
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,3,0.007829333345095316
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,131071,1.6749226252237956
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,7,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,131071,5.8163197835286455
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,15,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,15,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,31,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,63,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,127,0.008325333396593729
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,127,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,255,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,255,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,511,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,511,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,1023,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,1023,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,2047,0.034815999368826546
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,2047,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,4095,0.05751466751098633
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,4095,0.07441066702206929
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,8191,0.21282132466634116
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,8191,0.13960533340771994
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,16383,0.4210346539815267
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,16383,0.27477333943049115
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,32767,0.8436053593953451
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,32767,0.5374293327331543
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,65535,1.0617173512776692
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,65535,1.683626651763916
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,1,0.007349333415428798
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,1,0.010944000134865442
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,3,0.00749333327015241
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,131071,2.112511952718099
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,131071,3.3612801233927407
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,3,0.010944000134865442
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,7,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,15,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,15,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,31,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,63,0.008197333042820295
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,31,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,63,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,127,0.008709333216150602
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,127,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,255,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,511,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,511,0.0315733328461647
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,1023,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,1023,0.04417600234349569
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,2047,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,2047,0.0653653343518575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,4095,0.025770666698614757
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,4095,0.10922666390736897
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,8191,0.03703466554482778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,8191,0.19660800695419312
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,16383,0.0580266664425532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,16383,0.3705173333485921
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,32767,0.20121600230534872
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,32767,0.7195200125376383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,65535,0.3887786865234375
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,65535,1.4595413208007812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,131071,0.7731200059254965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,7,0.007327999919652939
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,131071,2.913450558980306
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,31,0.007461333026488622
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,31,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,63,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,63,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,127,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,255,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,255,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,511,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,511,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,1023,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,1023,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,2047,0.026965332527955372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,2047,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,4095,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,4095,0.040618665516376495
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,8191,0.0576853354771932
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,8191,0.07321600119272868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,16383,0.2001919945081075
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,16383,0.13858133554458618
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,32767,0.38843735059102374
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,32767,0.2725546757380168
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,65535,0.7685120105743408
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,65535,0.5343573490778605
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,1,0.007109333450595538
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,1,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,7,0.007151999821265538
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,131071,1.5272960662841797
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,131071,1.061205307642619
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,15,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,31,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,31,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,63,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,63,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,127,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,255,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,127,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,255,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,511,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,511,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,1023,0.011551999797423681
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,1023,0.018746666610240936
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,2047,0.01826133330663045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,2047,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,4095,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,4095,0.019285333653291065
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,8191,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,8191,0.0194560003777345
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,16383,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,16383,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,32767,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,32767,0.053247998158137
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,65535,0.019626667102177937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,65535,0.09489066402117412
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,131071,0.027818667391935985
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,131071,0.17934399843215942
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,1,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,3,0.010949333508809408
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,7,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,15,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,31,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,31,0.007285333548982938
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,63,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,63,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,127,0.008693333094318708
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,127,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,255,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,255,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,511,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,511,0.01791999985774358
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,1023,0.011322667201360067
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,1023,0.019968000551064808
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,2047,0.012063999970753988
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,2047,0.01911466692884763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,4095,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,4095,0.019968000551064808
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,8191,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,8191,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,16383,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,16383,0.018778666853904724
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,32767,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,32767,0.053930665055910744
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,65535,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,65535,0.09591466188430786
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,131071,0.040106666584809623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,131071,0.1800533334414164
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,3,0.007205333560705185
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,7,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,7,0.007125333572427432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,15,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,15,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,31,0.007189333438873291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,31,0.01180800050497055
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,63,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,63,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,127,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,255,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,255,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,511,0.00847999999920527
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,511,0.018079999834299088
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,1023,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,1023,0.018911999960740406
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,2047,0.012122667084137598
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,2047,0.019285333653291065
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,4095,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,4095,0.019626667102177937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,8191,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,8191,0.018778666853904724
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,16383,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,16383,0.053077335158983864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,32767,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,32767,0.09506133198738098
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,65535,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,65535,0.17783466974894205
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,131071,0.040106666584809623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,131071,0.33107199271519977
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,3,0.00731733317176501
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,3,0.011946666985750198
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,7,0.00701333334048589
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,7,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,15,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,15,0.011253333340088526
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,31,0.007354666789372762
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,31,0.011946666985750198
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,63,0.008010666817426682
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,63,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,127,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,255,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,511,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,511,0.018090666582187016
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,1023,0.011413333316644033
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,1023,0.020138667275508244
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,2047,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,2047,0.019626667102177937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,4095,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,4095,0.018090666582187016
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,8191,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,8191,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,16383,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,16383,0.05376000205675761
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,32767,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,32767,0.09596266349156697
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,65535,0.032085334261258446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,65535,0.1784906586011251
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,131071,0.05597866574923197
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,1,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,131071,0.33209067583084106
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,3,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,7,0.007477333148320516
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,7,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,31,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,31,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,63,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,63,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,127,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,255,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,511,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,511,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,1023,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,1023,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,2047,0.026965332527955372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,2047,0.024746666351954143
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,4095,0.04095999896526337
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,4095,0.040618665516376495
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,8191,0.0576853354771932
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,8191,0.07389866809050243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,16383,0.1996799906094869
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,16383,0.1389226714769999
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,32767,0.3891199827194214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,32767,0.2727253238360087
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,1,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,3,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,65535,0.7732906341552734
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,65535,0.5346986850102743
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,7,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,31,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,31,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,63,0.0074560002734263735
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,255,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,511,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,511,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,1023,0.028160000840822857
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,1023,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,2047,0.040106666584809623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,2047,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,4095,0.05717333157857259
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,4095,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,8191,0.20002132654190063
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,8191,0.07406933108965556
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,16383,0.39031465848286945
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,16383,0.1945599913597107
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,1,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,1,0.010954666882753372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,32767,0.7712426980336508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,32767,0.37717334429423016
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,3,0.007162666569153468
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,3,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,65535,1.5397547086079915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,65535,0.7384746869405111
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,7,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,7,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,15,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,31,0.0075040000180403394
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,31,0.012128000458081564
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,63,0.007738666608929634
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,63,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,127,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,255,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,255,0.014170666535695394
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,511,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,511,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,1023,0.01145600030819575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,1023,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,2047,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,2047,0.019813333948453266
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,4095,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,4095,0.031231999397277832
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,8191,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,8191,0.052784000833829246
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,16383,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,16383,0.09522133072217305
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,32767,0.025770666698614757
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,32767,0.17851734161376953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,65535,0.03276800115903219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,65535,0.32972800731658936
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,131071,0.056320001681645714
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,131071,0.6357333262761434
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,3,0.0069973332186539965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,7,0.007205333560705185
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,7,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,15,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,15,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,31,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,31,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,63,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,63,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,127,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,255,0.008133333176374435
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,255,0.015216000378131866
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,511,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,511,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,1023,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,1023,0.019968000551064808
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,2047,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,2047,0.018432000031073887
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,4095,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,4095,0.032085334261258446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,8191,0.01791999985774358
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,8191,0.053930665055910744
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,16383,0.022015998760859173
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,16383,0.09608532985051473
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,32767,0.034474665919939675
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,32767,0.17890133460362753
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,65535,0.05478399991989136
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,65535,0.3304106593132019
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,1,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,131071,0.20770132541656494
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,131071,0.6365866661071777
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,3,0.0074879998962084455
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,7,0.007466666400432587
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,31,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,63,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,127,0.008506666868925095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,255,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,511,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,511,0.01826133330663045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,1023,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,1023,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,2047,0.040106666584809623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,2047,0.024746666351954143
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,4095,0.0576853354771932
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,4095,0.04113066693147024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,8191,0.07406933108965556
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,8191,0.22886399428049722
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,16383,0.39236267407735187
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,16383,0.19592533508936563
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,1,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,1,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,3,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,3,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,32767,0.777557373046875
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,32767,0.3778560161590576
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,7,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,7,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,15,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,31,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,31,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,63,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,63,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,127,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,127,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,255,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,255,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,511,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,511,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,1023,0.03908266623814901
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,1023,0.025770666698614757
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,2047,0.05734399954477946
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,2047,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,4095,0.20087466637293497
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,4095,0.07628799974918365
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,8191,0.39150933424631756
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,8191,0.1955839991569519
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,1,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,1,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,16383,0.7732906341552734
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,16383,0.37512532869974774
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,3,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,3,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,32767,1.5402666727701824
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,32767,0.7301119963328043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,7,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,7,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,15,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,15,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,31,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,31,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,63,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,63,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,127,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,127,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,255,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,255,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,511,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,511,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,1023,0.03908266623814901
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,1023,0.025770666698614757
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,2047,0.05717333157857259
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,2047,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,4095,0.20155733823776245
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,4095,0.0766293356815974
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,1,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,1,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,3,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,3,0.016895999511082966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,8191,0.1967786749204
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,8191,0.3944106499354045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,7,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,7,0.016895999511082966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,15,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,15,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,31,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,31,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,63,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,63,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,127,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,127,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,255,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,255,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,511,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,511,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,1023,0.056832000613212585
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,1023,0.04659200211366018
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,2047,0.20189867417017618
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,2047,0.08072533210118611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,4095,0.1955839991569519
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,4095,0.392192006111145
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,1,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,1,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,3,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,3,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,8191,0.7770453294118246
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,8191,0.37597866853078205
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,7,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,7,0.016895999511082966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,15,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,15,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,31,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,31,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,63,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,63,0.016895999511082966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,127,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,127,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,255,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,255,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,511,0.040448000033696495
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,511,0.031231999397277832
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,1023,0.046762665112813316
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,1023,0.05717333157857259
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,2047,0.20309333006540933
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,2047,0.08089600006739299
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,1,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,1,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,3,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,3,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,4095,0.39560532569885254
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,4095,0.19729065895080566
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,7,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,7,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,15,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,15,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,31,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,31,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,63,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,63,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,127,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,127,0.03532800078392029
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,255,0.03703466554482778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,255,0.04232533276081085
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,511,0.053247998158137
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,511,0.057002668579419456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,1023,0.20002132654190063
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,1023,0.0865280032157898
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,2047,0.3954346577326457
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,2047,0.19933867454528809
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,1,0.007162666569153468
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,1,0.011461333682139715
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,3,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,4095,0.7796053091684977
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,4095,0.3848533233006795
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,15,0.007029333462317784
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,7,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,15,0.01246400053302447
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,31,0.007301333049933116
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,31,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,63,0.008000000069538752
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,127,0.008559999987483025
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,63,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,255,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,255,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,511,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,511,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,1023,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,1023,0.018464000274737675
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,2047,0.03925333420435587
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,2047,0.032085334261258446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,4095,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,4095,0.053930665055910744
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,8191,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,8191,0.091648002465566
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,16383,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,16383,0.15735466281572977
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,32767,0.03498666733503342
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,32767,0.2930346727371216
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,65535,0.05444266895453135
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,65535,0.5766666730244955
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,1,0.011461333682139715
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,131071,0.20821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,3,0.007296000296870868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,131071,0.7210666338602701
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,3,0.011952000359694162
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,7,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,15,0.007205333560705185
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,15,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,31,0.012170666207869848
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,31,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,63,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,63,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,127,0.009056000038981438
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,127,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,255,0.008346666892369589
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,255,0.013967999567588171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,511,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,511,0.019306667149066925
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,1023,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,1023,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,2047,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,2047,0.03242666771014532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,4095,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,4095,0.05407999952634176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,8191,0.091648002465566
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,8191,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,16383,0.03583999971548716
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,16383,0.15786666671435037
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,32767,0.05478399991989136
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,32767,0.29337600866953534
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,65535,0.20428800582885742
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,65535,0.5773653189341227
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,1,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,1,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,131071,0.3997013171513875
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,131071,0.7393279870351156
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,3,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,3,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,7,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,7,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,15,0.028160000840822857
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,15,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,31,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,63,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,31,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,63,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,127,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,127,0.03566933423280716
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,255,0.04232533276081085
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,255,0.03703466554482778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,511,0.052906667192777
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,511,0.056661332647005715
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,1023,0.08703999718030293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,1023,0.22476800282796225
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,1,0.05000533163547516
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,1,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,3,0.05000533163547516
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,3,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,2047,0.40192000071207684
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,7,0.05034666756788889
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,7,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,2047,0.2001919945081075
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,15,0.04966400067011515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,15,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,31,0.04966400067011515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,31,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,63,0.07850666840871175
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,63,0.05034666756788889
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,127,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,127,0.06365866462389629
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,255,0.06587733328342438
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,255,0.07850666840871175
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,511,0.19780266284942627
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,511,0.10478933652242024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,1023,0.19490132729212442
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,1023,0.38929065068562824
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,1,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,1,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,3,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,3,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,2047,0.37649067242940265
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,2047,0.772266705830892
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,7,0.05034666756788889
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,7,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,15,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,15,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,31,0.05034666756788889
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,31,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,63,0.05034666756788889
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,63,0.05085866649945577
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,127,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,127,0.06400000055631001
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,255,0.06587733328342438
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,255,0.07867733140786488
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,1,0.09335466225941975
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,1,0.09540266791979472
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,511,0.20155733823776245
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,511,0.10410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,3,0.09352533022562663
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,3,0.09540266791979472
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,7,0.09352533022562663
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,7,0.09540266791979472
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,15,0.09540266791979472
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,1023,0.3954346577326457
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,31,0.09352533022562663
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,15,0.09352533022562663
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,31,0.09540266791979472
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,1023,0.20053333044052124
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,63,0.09335466225941975
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,63,0.09540266791979472
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,127,0.09437867005666097
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,127,0.1220266620318095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,255,0.1971199909845988
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,255,0.15069866180419922
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,1,0.01108266661564509
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,3,0.011285333583752314
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,511,0.21657600005467734
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,511,0.38758401075998944
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,7,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,15,0.01128000020980835
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,31,0.012144000579913458
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,31,0.007823999971151352
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,63,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,63,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,1023,0.7674880027770996
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,255,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,255,0.014314666390419006
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,1023,0.37700267632802326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,127,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,511,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,511,0.018944000204404194
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,1023,0.010549332946538925
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,1023,0.03259733319282532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,2047,0.04898133377234141
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,2047,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,4095,0.07543466488520305
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,8191,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,4095,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,8191,0.1358506679534912
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,16383,0.03549866626660029
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,16383,0.2505386670430501
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,32767,0.055125330885251365
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,32767,0.4843093156814575
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,65535,0.2032639980316162
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,1,0.007360000163316727
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,65535,0.7195146878560384
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,131071,0.3990186850229899
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,131071,1.4622507095336914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,3,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,7,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,7,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,15,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,15,0.011312000453472137
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,63,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,31,0.0074879998962084455
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,31,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,63,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,127,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,127,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,255,0.013690666606028875
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,511,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,1023,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,1023,0.032618666688601174
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,511,0.01791999985774358
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,2047,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,2047,0.048469334840774536
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,4095,0.025429333249727886
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,4095,0.07577600081761678
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,8191,0.03754666695992152
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,8191,0.13567999998728433
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,16383,0.0580266664425532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,16383,0.25033066670099896
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,32767,0.20053333044052124
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,32767,0.48469332853953045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,65535,0.38980265458424884
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,65535,0.7377920150756836
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,1,0.00720000018676122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,1,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,3,0.007631999750932057
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,3,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,131071,0.7691946824391683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,7,0.007327999919652939
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,131071,1.4683039983113606
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,7,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,31,0.008410666758815447
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,15,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,31,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,63,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,63,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,127,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,255,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,255,0.03310933212439219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,511,0.045594667394955955
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,511,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,1023,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,1023,0.06724266707897186
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,2047,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,2047,0.11246933539708455
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,4095,0.03328000009059906
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,4095,0.20172800620396933
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,8191,0.04949333270390829
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,8191,0.3781973520914714
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,16383,0.21623466412226358
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,16383,0.7313066323598226
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,32767,0.4184746742248535
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,32767,1.4569813410441081
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,65535,0.8343893686930338
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,1,0.008352000266313553
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,65535,2.9071359634399414
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,3,0.007642666498819987
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,131071,1.666218598683675
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,7,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,7,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,15,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,131071,5.807104110717773
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,31,0.007674666742483775
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,63,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,127,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,255,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,511,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,511,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,255,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,1023,0.022015998760859173
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,1023,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,2047,0.03515733281771342
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,2047,0.04215466479460398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,4095,0.056832000613212585
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,4095,0.07492266595363617
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,8191,0.21248000860214233
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,8191,0.14045866330464682
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,16383,0.4357120196024577
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,16383,0.2754559914271037
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,32767,0.8313173453013102
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,32767,0.5382826725641886
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,65535,1.6561493873596191
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,1,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,65535,1.062058687210083
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,3,0.007007999966541926
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,3,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,131071,3.299327850341797
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,131071,2.112170696258545
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,7,0.00731733317176501
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,7,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,15,0.013674666484196981
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,31,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,31,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,127,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,127,0.014671999961137772
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,255,0.020138667275508244
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,511,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,511,0.021162666380405426
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,1023,0.020138667275508244
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,1023,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,2047,0.012432000289360682
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,2047,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,4095,0.019952000429232914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,4095,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,8191,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,8191,0.020138667275508244
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,16383,0.0543039987484614
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,16383,0.016042667130629223
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,32767,0.019626667102177937
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,32767,0.09710933764775594
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,65535,0.024405332903067272
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,65535,0.18120533227920532
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,131071,0.0339626669883728
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,131071,0.3387840191523234
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,1,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,3,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,7,0.007135999699433644
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,7,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,15,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,31,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,31,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,63,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,127,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,127,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,255,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,255,0.019797333826621372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,511,0.011546666423479715
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,511,0.021503999829292297
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,1023,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,1023,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,2047,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,2047,0.020997333029905956
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,4095,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,4095,0.020138667275508244
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,8191,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,8191,0.020469332734743755
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,16383,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,16383,0.05563733478387197
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,32767,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,32767,0.0981333355108897
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,65535,0.028330666323502857
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,65535,0.18227199713389078
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,1,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,131071,0.053247998158137
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,131071,0.34011733531951904
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,3,0.01314666618903478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,7,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,15,0.013658666362365087
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,31,0.007791999727487564
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,31,0.014837333311637243
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,63,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,127,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,255,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,255,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,511,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,511,0.02110933264096578
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,1023,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,2047,0.012597333639860153
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,2047,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,1023,0.021503999829292297
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,4095,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,4095,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,8191,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,8191,0.05444266895453135
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,16383,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,16383,0.09657599528630574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,32767,0.021162666380405426
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,65535,0.028160000840822857
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,32767,0.1812480092048645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,65535,0.33639999230702716
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,131071,0.05239466826121012
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,131071,0.6478506724039713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,1,0.007162666569153468
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,1,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,3,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,7,0.007120000198483467
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,7,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,15,0.013999999811251959
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,31,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,31,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,63,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,127,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,127,0.008538666491707167
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,255,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,255,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,511,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,511,0.021162666380405426
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,1023,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,1023,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,2047,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,2047,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,4095,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,4095,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,8191,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,8191,0.05563200016816457
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,16383,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,16383,0.09796266754468282
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,32767,0.03276800115903219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,32767,0.18210132916768393
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,65535,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,65535,0.33745598793029785
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,131071,0.23091200987497965
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,1,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,1,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,3,0.0074879998962084455
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,131071,0.6493653456370035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,7,0.008010666817426682
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,3,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,15,0.008309333274761835
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,63,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,255,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,255,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,511,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,511,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,1023,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,1023,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,2047,0.04215466479460398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,2047,0.034474665919939675
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,4095,0.05649066468079885
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,4095,0.07509333391984303
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,8191,0.2136746644973755
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,8191,0.14045866330464682
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,16383,0.27562665939331055
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,16383,0.4194986820220947
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,32767,0.8289279937744141
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,1,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,32767,0.5382826725641886
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,3,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,65535,1.662805398305257
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,65535,1.0627413590749104
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,7,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,15,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,31,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,63,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,63,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,255,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,255,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,511,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,511,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,1023,0.03532800078392029
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,1023,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,2047,0.055125330885251365
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,2047,0.04232533276081085
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,4095,0.2135039965311686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,4095,0.07509333391984303
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,8191,0.41762133439381915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,8191,0.21555199225743613
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,16383,0.8275626500447592
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,16383,0.41727999846140545
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,32767,0.8214186827341715
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,1,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,32767,1.6457386016845703
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,3,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,3,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,65535,1.6266239484151204
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,7,0.007311999797821045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,65535,3.287210782368978
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,7,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,15,0.015008000036080679
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,31,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,31,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,63,0.008341333518425623
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,63,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,127,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,255,0.019461333751678467
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,511,0.011637333780527115
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,511,0.020992000897725422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,1023,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,1023,0.020288000504175823
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,2047,0.032255999743938446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,2047,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,4095,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,4095,0.054272000988324486
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,8191,0.0170666662355264
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,8191,0.09691733121871948
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,16383,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,16383,0.18141865730285645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,32767,0.03293866664171219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,32767,0.33617067337036133
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,65535,0.04454400142033895
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,65535,0.6447786490122477
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,131071,0.22732800245285034
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,131071,1.269759972890218
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,1,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,3,0.007173333317041397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,7,0.007311999797821045
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,3,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,7,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,15,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,15,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,31,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,31,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,127,0.016042667130629223
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,127,0.00949866697192192
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,255,0.019797333826621372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,511,0.011946666985750198
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,511,0.020992000897725422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,1023,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,1023,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,2047,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,2047,0.03362133353948593
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,4095,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,4095,0.05513600011666616
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,8191,0.023381332556406658
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,8191,0.0981333355108897
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,16383,0.02867199977238973
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,16383,0.18227734168370566
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,32767,0.04539733131726583
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,32767,0.3370613257090251
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,65535,0.24081067244211832
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,65535,0.6458026568094889
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,1,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,1,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,131071,0.44014934698740643
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,3,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,131071,1.3050933678944905
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,3,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,7,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,7,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,15,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,15,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,31,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,31,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,63,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,127,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,255,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,255,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,511,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,511,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,1023,0.03583999971548716
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,1023,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,2047,0.05478399991989136
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,2047,0.04215466479460398
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,4095,0.21555199225743613
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,4095,0.07509333391984303
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,8191,0.21486934026082358
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,8191,0.41864534219106037
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,16383,0.8292693297068278
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,16383,0.41250133514404297
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,1,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,1,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,3,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,3,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,32767,1.652224063873291
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,32767,0.8023040294647217
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,7,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,15,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,15,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,7,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,31,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,31,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,63,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,63,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,127,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,127,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,255,0.02218666672706604
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,255,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,511,0.0363520011305809
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,511,0.02679466704527537
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,1023,0.05222400029500326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,1023,0.043178667624791466
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,2047,0.21401600042978922
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,2047,0.07816533247629802
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,4095,0.41915734608968097
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,4095,0.21657600005467734
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,8191,0.8296106656392416
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,8191,0.41710933049519855
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,16383,1.6493226687113445
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,1,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,16383,0.8144213358561198
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,1,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,3,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,3,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,32767,3.2822612126668296
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,7,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,7,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,32767,1.594879945119222
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,15,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,15,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,31,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,31,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,63,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,63,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,127,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,127,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,255,0.02218666672706604
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,511,0.0363520011305809
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,255,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,511,0.026965332527955372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,1023,0.05239466826121012
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,1023,0.043178667624791466
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,2047,0.2152106761932373
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,2047,0.07799466451009114
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,4095,0.42052265008290607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,4095,0.22152533133824667
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,1,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,1,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,8191,0.8277333577473959
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,3,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,3,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,8191,0.41881601015726727
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,7,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,7,0.019797333826621372
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,15,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,15,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,31,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,63,0.017407999684413273
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,63,0.019968000551064808
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,31,0.019968000551064808
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,127,0.021503999829292297
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,127,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,255,0.036864000062147774
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,255,0.02935466667016347
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,511,0.053930665055910744
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,511,0.049322664737701416
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,1023,0.21128533283869425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,1023,0.0820906658967336
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,2047,0.4220586617787679
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,2047,0.2198186715443929
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,4095,0.8328533172607422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,4095,0.41045331954956055
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,1,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,1,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,8191,1.6604159673055012
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,3,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,3,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,8191,0.8028159936269125
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,7,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,7,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,15,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,15,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,31,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,31,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,63,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,63,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,127,0.020992000897725422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,127,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,255,0.037205333511034645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,255,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,511,0.05358933409055074
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,511,0.049322664737701416
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,1023,0.2136746644973755
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,1023,0.0820906658967336
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,2047,0.4241066773732503
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,2047,0.2249386707941691
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,1,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,1,0.0365226666132609
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,3,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,3,0.045909335215886436
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,4095,0.8360959688822428
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,7,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,7,0.03618133316437403
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,4095,0.4196693499883016
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,15,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,15,0.0363520011305809
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,31,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,31,0.0363520011305809
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,63,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,63,0.03703466554482778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,127,0.03703466554482778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,127,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,255,0.04693333307902018
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,255,0.053247998158137
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,511,0.2141866683959961
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,511,0.09250133236249287
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,1023,0.2218666672706604
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,1023,0.42871467272440594
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,1,0.007285333548982938
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,2047,0.8538453578948975
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,1,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,2047,0.423253337542216
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,3,0.00701333334048589
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,3,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,4095,1.6904533704121907
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,7,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,4095,0.8256853421529134
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,15,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,15,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,31,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,31,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,63,0.014885333677132925
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,127,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,255,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,511,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,255,0.019285333653291065
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,511,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,1023,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,1023,0.0339626669883728
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,2047,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,2047,0.05563733478387197
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,4095,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,4095,0.0936959981918335
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,8191,0.023210667073726654
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,8191,0.1614773372809092
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,16383,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,32767,0.04437333345413208
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,16383,0.30004799365997314
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,32767,0.5862506628036499
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,65535,0.2208426594734192
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,65535,1.1528533299763997
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,1,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,131071,0.429909348487854
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,1,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,3,0.008703999842206636
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,131071,1.4619305928548176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,3,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,7,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,7,0.013269333789745966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,15,0.007397333160042763
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,31,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,15,0.013637332866589228
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,63,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,31,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,127,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,127,0.009359999870260557
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,255,0.008538666491707167
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,255,0.019968000551064808
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,511,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,511,0.020341333001852036
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,1023,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,1023,0.0339626669883728
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,2047,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,2047,0.0554666668176651
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,4095,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,4095,0.09386666615804036
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,8191,0.16142400105794272
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,8191,0.032085334261258446
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,16383,0.045738667249679565
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,16383,0.29973334074020386
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,32767,0.21998933951059976
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,32767,0.5860693454742432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,65535,0.42922667662302655
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,65535,1.184938669204712
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,1,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,1,0.03703466554482778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,131071,0.8574293454488119
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,131071,1.4820693333943684
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,3,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,3,0.037205333511034645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,7,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,7,0.037205333511034645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,15,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,15,0.03703466554482778
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,31,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,63,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,31,0.037205333511034645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,63,0.037205333511034645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,127,0.037205333511034645
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,127,0.041984001795450844
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,255,0.05256533126036326
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,255,0.05376000205675761
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,511,0.09215999643007915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,511,0.21760000785191855
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,1023,0.4331520001093547
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,1,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,1023,0.22510933876037598
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,1,0.06587733328342438
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,3,0.05034666756788889
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,3,0.06587733328342438
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,2047,0.8587946891784668
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,7,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,7,0.06587733328342438
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,15,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,2047,0.43383467197418213
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,15,0.06587733328342438
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,31,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,31,0.06587733328342438
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,63,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,63,0.06587733328342438
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,127,0.06604800124963124
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,127,0.07526400188604991
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,255,0.09710933764775594
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,255,0.20923733711242676
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,511,0.41779200236002606
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,511,0.21077332894007364
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,1023,0.839850664138794
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,1,0.051541333397229515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,1023,0.4140373468399048
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,1,0.06673066814740498
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,3,0.051541333397229515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,3,0.06690133114655812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,2047,1.6767999331156414
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,2047,0.8272213141123453
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,7,0.051541333397229515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,7,0.06690133114655812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,15,0.051541333397229515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,15,0.06673066814740498
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,31,0.051541333397229515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,31,0.06673066814740498
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,63,0.051541333397229515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,63,0.06690133114655812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,127,0.06690133114655812
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,127,0.07628799974918365
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,255,0.2146986722946167
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,255,0.0981333355108897
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,511,0.42632532119750977
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,1,0.09523199995358785
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,1,0.12424533565839131
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,3,0.09506133198738098
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,3,0.12424533565839131
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,511,0.21811199188232422
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,7,0.09506133198738098
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,7,0.12424533565839131
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,15,0.09523199995358785
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,1023,0.8490666548411051
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,1023,0.4230826695760091
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,15,0.12424533565839131
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,31,0.09523199995358785
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,31,0.12424533565839131
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,63,0.09642666578292847
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,63,0.12424533565839131
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,127,0.20036266247431436
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,127,0.14472533265749613
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,255,0.41147732734680176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,255,0.21145600080490112
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,1,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,1,0.01333333303531011
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,3,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,511,0.8232959906260172
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,7,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,511,0.40465064843495685
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,7,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,15,0.007344000041484833
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,15,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,31,0.008021333565314611
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,31,0.014688000082969666
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,63,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,1023,1.657002607981364
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,127,0.0085333331177632
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,255,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,1023,0.7990612983703613
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,255,0.0199946661790212
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,511,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,511,0.034474665919939675
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,1023,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,1023,0.05034666756788889
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,2047,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,2047,0.0782239983479182
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,4095,0.021333334346612293
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,4095,0.13960533340771994
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,8191,0.031744000812371574
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,8191,0.25621867179870605
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,16383,0.04539733131726583
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,16383,0.4928853511810303
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,32767,0.21742933988571167
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,32767,0.9615360101064047
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,65535,0.4217173258463542
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,1,0.007338666667540868
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,65535,1.4591840108235676
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,1,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,131071,0.8386560281117758
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,3,0.007167999943097432
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,131071,2.9149866104125977
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,3,0.013007999708255133
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,7,0.007360000163316727
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,7,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,15,0.00766933336853981
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,15,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,31,0.007850666840871176
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,31,0.014688000082969666
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,63,0.008362666393319765
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,63,0.01540800059835116
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,127,0.008389333263039589
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,127,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,255,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,255,0.019285333653291065
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,511,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,511,0.034458667039871216
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,1023,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,1023,0.050399998823801674
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,2047,0.07833600044250488
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,2047,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,4095,0.03310933212439219
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,4095,0.13926399747530618
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,8191,0.04966400067011515
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,8191,0.2563413381576538
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,16383,0.2172586719195048
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,16383,0.4930400053660075
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,32767,0.4212053219477336
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,32767,0.9849173227945963
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,65535,0.8391679922739664
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,65535,1.480533281962077
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,131071,1.666218598683675
TRTLLM,1.0.0,NVIDIA L40S,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,131071,2.9482666651407876
