framework,version,device,op_name,kernel_source,mla_dtype,kv_cache_dtype,num_heads,batch_size,isl,tp_size,step,latency
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1,1,1,1,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1,1,2,1,0.011152000476916632
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1,1,8,1,0.010543999572594961
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1,1,4,1,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1,1,32,1,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,1,1,64,1,0.0102186668664217
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1,1,1,1,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1,1,4,1,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1,1,8,1,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1,1,2,1,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1,1,16,1,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1,1,32,1,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,1,1,64,1,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1,1,16,1,0.01003200002014637
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1,1,4,3,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1,1,8,3,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1,1,1,3,0.011946666985750198
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1,1,16,3,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1,1,32,3,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1,1,2,3,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1,1,2,3,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1,1,1,3,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1,1,4,3,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1,1,16,3,0.010575999816258749
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,1,1,64,3,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1,1,8,3,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1,1,32,3,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,1,1,64,3,0.010384000216921171
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1,1,1,7,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1,1,4,7,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1,1,8,7,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1,1,2,7,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1,1,16,7,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,1,1,64,7,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1,1,1,7,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1,1,32,7,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1,1,8,7,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1,1,4,7,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1,1,2,7,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1,1,16,7,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1,1,32,7,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1,1,1,15,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1,1,2,15,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,1,1,64,7,0.010389333590865135
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1,1,4,15,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1,1,8,15,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1,1,16,15,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,1,1,64,15,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1,1,32,15,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1,1,4,15,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1,1,2,15,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1,1,1,15,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1,1,8,15,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1,1,32,15,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1,1,16,15,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1,1,1,31,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,1,1,64,15,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1,1,2,31,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1,1,4,31,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1,1,8,31,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1,1,16,31,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1,1,32,31,0.010368000095089277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,1,1,64,31,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1,1,1,31,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1,1,2,31,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1,1,8,31,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1,1,16,31,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1,1,4,31,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,1,1,64,31,0.0103946669648091
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1,1,32,31,0.010389333590865135
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1,1,1,63,0.011258666714032492
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1,1,2,63,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1,1,4,63,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1,1,16,63,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1,1,8,63,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,1,1,64,63,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1,1,1,63,0.04215466479460398
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1,1,32,63,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1,1,4,63,0.010575999816258749
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1,1,2,63,0.014639999717473984
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1,1,8,63,0.01452800010641416
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,1,1,64,63,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1,1,32,63,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1,1,16,63,0.010384000216921171
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1,1,1,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1,1,2,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1,1,4,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1,1,8,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1,1,16,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1,1,32,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,1,1,64,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1,1,2,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1,1,1,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1,1,4,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1,1,16,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1,1,32,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1,1,8,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,1,1,64,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1,1,1,255,0.024405332903067272
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1,1,2,255,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1,1,4,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1,1,16,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1,1,8,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1,1,32,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,1,1,64,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1,1,1,255,0.024405332903067272
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1,1,2,255,0.024405332903067272
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1,1,8,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1,1,4,255,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1,1,16,255,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,1,1,64,255,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1,1,32,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1,1,2,511,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1,1,1,511,0.04488533238569895
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1,1,4,511,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1,1,8,511,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1,1,16,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1,1,32,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,1,1,64,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1,1,1,511,0.04437333345413208
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1,1,4,511,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1,1,2,511,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1,1,8,511,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1,1,16,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1,1,32,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,1,1,64,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1,1,1,1023,0.08413867155710857
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1,1,2,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1,1,4,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1,1,8,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1,1,16,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1,1,32,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,1,1,64,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1,1,1,1023,0.08396800359090169
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1,1,2,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1,1,4,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1,1,8,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1,1,16,1023,0.08345599969228108
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1,1,32,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,1,1,64,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1,1,1,2047,0.16366933782895407
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1,1,2,2047,0.16332800189654031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1,1,8,2047,0.16315733393033346
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1,1,4,2047,0.16315733393033346
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1,1,16,2047,0.16332800189654031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1,1,32,2047,0.16315733393033346
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,1,1,64,2047,0.16315733393033346
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1,1,1,2047,0.1634986698627472
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1,1,2,2047,0.16332800189654031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1,1,4,2047,0.16315733393033346
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1,1,16,2047,0.1629866659641266
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1,1,8,2047,0.16315733393033346
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1,1,32,2047,0.16315733393033346
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,1,1,64,2047,0.16315733393033346
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1,1,1,4095,0.32238932450612384
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1,1,2,4095,0.3222186764081319
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1,1,4,4095,0.3222186764081319
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1,1,8,4095,0.3222186764081319
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1,1,16,4095,0.32204800844192505
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1,1,32,4095,0.3218773404757182
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,1,1,64,4095,0.32238932450612384
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1,1,1,4095,0.32255999247233075
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1,1,2,4095,0.3222186764081319
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1,1,4,4095,0.3222186764081319
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1,1,8,4095,0.3222186764081319
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1,1,16,4095,0.3218773404757182
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1,1,32,4095,0.32204800844192505
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,1,1,64,4095,0.32238932450612384
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1,1,1,8191,0.6411946614583334
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1,1,2,8191,0.6401706536610922
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1,1,4,8191,0.6399999856948853
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1,1,8,8191,0.6398293177286783
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1,1,16,8191,0.6398293177286783
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,1,1,64,8191,0.6399999856948853
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1,1,32,8191,0.6398293177286783
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1,1,1,8191,0.640341321627299
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1,1,2,8191,0.6408533255259196
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1,1,4,8191,0.6399999856948853
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1,1,8,8191,0.6399999856948853
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1,1,16,8191,0.6398293177286783
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,1,1,64,8191,0.6399999856948853
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1,1,32,8191,0.6398293177286783
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1,1,1,16383,1.2760746479034424
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1,1,2,16383,1.2759040196736653
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1,1,4,16383,1.2779520352681477
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1,1,8,16383,1.2760746479034424
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1,1,16,16383,1.2757333119710286
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1,1,32,16383,1.2757333119710286
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,1,1,64,16383,1.2759040196736653
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1,1,1,16383,1.276245355606079
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1,1,2,16383,1.2759040196736653
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1,1,4,16383,1.2757333119710286
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1,1,8,16383,1.2776106993357341
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,1,1,64,16383,1.2755626837412517
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1,1,32,16383,1.2755626837412517
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1,1,1,32767,2.54856538772583
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1,1,4,32767,2.5477120081583657
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1,1,16,16383,1.2757333119710286
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1,1,2,32767,2.5478827158610025
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1,1,8,32767,2.5477120081583657
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1,1,16,32767,2.55078395207723
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1,1,32,32767,2.547541300455729
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,1,1,64,32767,2.547541300455729
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1,1,1,32767,2.5487359364827475
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1,1,2,32767,2.5477120081583657
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1,1,4,32767,2.5483946800231934
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1,1,8,32767,2.547541300455729
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1,1,16,32767,2.547541300455729
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1,1,32,32767,2.55078395207723
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,1,1,64,32767,2.547541300455729
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1,1,1,65535,5.0920108159383135
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1,1,2,65535,5.093034744262695
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1,1,4,65535,5.091498692830403
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1,1,16,65535,5.091327985127767
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1,1,8,65535,5.092864036560059
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,1,1,64,65535,5.098325411478679
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1,1,32,65535,5.091327985127767
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1,1,1,65535,5.091840108235677
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1,1,2,65535,5.09166940053304
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1,1,4,65535,5.092693328857422
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1,1,8,65535,5.091498692830403
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1,1,16,65535,5.091498692830403
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1,1,32,65535,5.092522621154785
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,1,1,64,65535,5.097642580668132
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1,1,1,131071,10.377728144327799
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1,1,2,131071,10.364757537841797
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1,1,4,131071,10.353663762410482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1,1,16,131071,10.343423843383789
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1,1,8,131071,10.362538655598959
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1,1,32,131071,10.348543802897135
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,1,1,64,131071,10.359807968139648
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1,1,1,131071,10.405888239542643
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1,1,2,131071,10.36305046081543
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1,1,8,131071,10.350421269734701
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1,1,4,131071,10.359807968139648
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1,1,16,131071,10.34444808959961
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,2,1,1,1,0.011045332998037338
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,2,1,2,1,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,2,1,4,1,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1,1,32,131071,10.361685434977213
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,1,1,64,131071,10.349568049112955
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,2,1,8,1,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,2,1,32,1,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,2,1,16,1,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,2,1,64,1,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,2,1,1,1,0.010741333166758219
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,2,1,2,1,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,2,1,4,1,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,2,1,8,1,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,2,1,16,1,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,2,1,32,1,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,2,1,1,3,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,2,1,64,1,0.011621333658695221
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,2,1,4,3,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,2,1,2,3,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,2,1,8,3,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,2,1,16,3,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,2,1,32,3,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,2,1,64,3,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,2,1,2,3,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,2,1,1,3,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,2,1,4,3,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,2,1,8,3,0.010384000216921171
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,2,1,32,3,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,2,1,64,3,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,2,1,16,3,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,2,1,1,7,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,2,1,2,7,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,2,1,8,7,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,2,1,4,7,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,2,1,16,7,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,2,1,32,7,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,2,1,1,7,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,2,1,64,7,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,2,1,2,7,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,2,1,4,7,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,2,1,8,7,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,2,1,16,7,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,2,1,32,7,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,2,1,64,7,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,2,1,1,15,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,2,1,2,15,0.010570666442314783
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,2,1,8,15,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,2,1,32,15,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,2,1,4,15,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,2,1,16,15,0.010399999717871347
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,2,1,2,15,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,2,1,64,15,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,2,1,1,15,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,2,1,4,15,0.010575999816258749
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,2,1,8,15,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,2,1,16,15,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,2,1,32,15,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,2,1,64,15,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,2,1,2,31,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,2,1,1,31,0.011087999989589056
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,2,1,4,31,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,2,1,8,31,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,2,1,16,31,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,2,1,32,31,0.010405333091815313
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,2,1,64,31,0.010351999973257383
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,2,1,1,31,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,2,1,2,31,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,2,1,8,31,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,2,1,16,31,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,2,1,32,31,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,2,1,64,31,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,2,1,1,63,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,2,1,2,63,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,2,1,4,31,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,2,1,4,63,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,2,1,16,63,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,2,1,8,63,0.010399999717871347
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,2,1,32,63,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,2,1,1,63,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,2,1,4,63,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,2,1,2,63,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,2,1,8,63,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,2,1,64,63,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,2,1,16,63,0.010213333492477735
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,2,1,32,63,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,2,1,1,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,2,1,64,63,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,2,1,2,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,2,1,4,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,2,1,8,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,2,1,16,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,2,1,32,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,2,1,64,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,2,1,1,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,2,1,2,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,2,1,4,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,2,1,8,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,2,1,16,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,2,1,32,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,2,1,64,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,2,1,1,255,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,2,1,2,255,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,2,1,4,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,2,1,8,255,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,2,1,32,255,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,2,1,16,255,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,2,1,64,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,2,1,2,255,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,2,1,1,255,0.024405332903067272
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,2,1,8,255,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,2,1,4,255,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,2,1,16,255,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,2,1,64,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,2,1,32,255,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,2,1,1,511,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,2,1,4,511,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,2,1,8,511,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,2,1,32,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,2,1,16,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,2,1,64,511,0.04369066655635834
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,2,1,1,511,0.04437333345413208
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,2,1,2,511,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,2,1,2,511,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,2,1,4,511,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,2,1,8,511,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,2,1,16,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,2,1,32,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,2,1,64,511,0.04369066655635834
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,2,1,1,1023,0.08413867155710857
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,2,1,2,1023,0.08396800359090169
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,2,1,8,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,2,1,16,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,2,1,32,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,2,1,64,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,2,1,1,1023,0.08413867155710857
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,2,1,2,1023,0.08396800359090169
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,2,1,4,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,2,1,4,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,2,1,8,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,2,1,32,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,2,1,16,1023,0.08345599969228108
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,2,1,64,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,2,1,1,2047,0.1634986698627472
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,2,1,2,2047,0.16366933782895407
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,2,1,4,2047,0.16315733393033346
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,2,1,16,2047,0.16315733393033346
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,2,1,32,2047,0.1629866659641266
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,2,1,64,2047,0.1629866659641266
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,2,1,2,2047,0.16332800189654031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,2,1,1,2047,0.16366933782895407
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,2,1,8,2047,0.16315733393033346
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,2,1,4,2047,0.1634986698627472
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,2,1,8,2047,0.16332800189654031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,2,1,16,2047,0.16315733393033346
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,2,1,32,2047,0.1629866659641266
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,2,1,64,2047,0.16315733393033346
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,2,1,2,4095,0.32238932450612384
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,2,1,1,4095,0.32255999247233075
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,2,1,32,4095,0.32204800844192505
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,2,1,8,4095,0.3227306604385376
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,2,1,16,4095,0.32204800844192505
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,2,1,64,4095,0.32204800844192505
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,2,1,1,4095,0.32255999247233075
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,2,1,4,4095,0.3222186764081319
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,2,1,2,4095,0.3222186764081319
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,2,1,4,4095,0.32204800844192505
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,2,1,8,4095,0.3222186764081319
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,2,1,16,4095,0.32238932450612384
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,2,1,64,4095,0.32204800844192505
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,2,1,32,4095,0.32204800844192505
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,2,1,1,8191,0.6405119895935059
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,2,1,8,8191,0.640341321627299
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,2,1,2,8191,0.640341321627299
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,2,1,4,8191,0.6401706536610922
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,2,1,16,8191,0.6399999856948853
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,2,1,32,8191,0.6406826575597128
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,2,1,64,8191,0.6398293177286783
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,2,1,1,8191,0.6405119895935059
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,2,1,2,8191,0.640341321627299
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,2,1,4,8191,0.6401706536610922
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,2,1,16,8191,0.6401706536610922
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,2,1,32,8191,0.6405119895935059
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,2,1,8,8191,0.6401706536610922
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,2,1,64,8191,0.6405119895935059
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,2,1,1,16383,1.2764159838358562
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,2,1,2,16383,1.276245355606079
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,2,1,4,16383,1.2760746479034424
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,2,1,32,16383,1.2760746479034424
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,2,1,8,16383,1.2760746479034424
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,2,1,16,16383,1.276245355606079
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,2,1,64,16383,1.2760746479034424
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,2,1,1,16383,1.2779520352681477
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,2,1,2,16383,1.276245355606079
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,2,1,4,16383,1.2760746479034424
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,2,1,8,16383,1.2760746479034424
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,2,1,32,16383,1.2759040196736653
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,2,1,64,16383,1.2759040196736653
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,2,1,16,16383,1.2760746479034424
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,2,1,1,32767,2.54856538772583
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,2,1,2,32767,2.551466623942057
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,2,1,8,32767,2.5483946800231934
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,2,1,4,32767,2.5483946800231934
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,2,1,16,32767,2.54805326461792
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,2,1,32,32767,2.54805326461792
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,2,1,1,32767,2.5487359364827475
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,2,1,64,32767,2.5478827158610025
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,2,1,2,32767,2.5483946800231934
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,2,1,4,32767,2.55129607518514
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,2,1,32,32767,2.54805326461792
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,2,1,16,32767,2.5482239723205566
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,2,1,8,32767,2.5482239723205566
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,2,1,64,32767,2.54805326461792
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,2,1,1,65535,5.199018796284993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,2,1,2,65535,5.186389287312825
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,2,1,4,65535,5.187413215637207
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,2,1,16,65535,5.183487892150879
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,2,1,8,65535,5.19048531850179
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,2,1,64,65535,5.182122548421224
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,2,1,32,65535,5.182976086934407
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,2,1,1,65535,5.203285217285156
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,2,1,2,65535,5.188437461853027
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,2,1,4,65535,5.182634671529134
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,2,1,8,65535,5.1858774820963545
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,2,1,32,65535,5.190314610799153
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,2,1,16,65535,5.185194651285808
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,2,1,64,65535,5.182122548421224
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,2,1,8,131071,10.359125137329102
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,2,1,4,131071,10.366122563680014
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,2,1,1,131071,10.40281613667806
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,2,1,2,131071,10.384725570678711
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,2,1,16,131071,10.367317199707031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,2,1,32,131071,10.374314626057943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,2,1,64,131071,10.364245096842447
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,2,1,1,131071,10.39189338684082
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,2,1,4,131071,10.36407470703125
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,4,1,1,1,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,2,1,2,131071,10.365439732869467
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,2,1,16,131071,10.367658615112305
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,2,1,8,131071,10.36458651224772
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,2,1,32,131071,10.369706471761068
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,2,1,64,131071,10.37380282084147
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,4,1,2,1,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,4,1,8,1,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,4,1,4,1,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,4,1,32,1,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,4,1,16,1,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,4,1,64,1,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,4,1,2,1,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,4,1,1,1,0.011077333241701126
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,4,1,4,1,0.010389333590865135
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,4,1,16,1,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,4,1,8,1,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,4,1,32,1,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,4,1,64,1,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,4,1,1,3,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,4,1,2,3,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,4,1,4,3,0.010197333370645842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,4,1,8,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,4,1,32,3,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,4,1,16,3,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,4,1,64,3,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,4,1,1,3,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,4,1,2,3,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,4,1,4,3,0.014293332894643148
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,4,1,16,3,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,4,1,8,3,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,4,1,32,3,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,4,1,1,7,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,4,1,64,3,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,4,1,2,7,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,4,1,4,7,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,4,1,8,7,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,4,1,32,7,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,4,1,64,7,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,4,1,1,7,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,4,1,16,7,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,4,1,2,7,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,4,1,4,7,0.010709332923094431
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,4,1,8,7,0.01055466632048289
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,4,1,16,7,0.01020800011853377
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,4,1,32,7,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,4,1,1,15,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,4,1,64,7,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,4,1,8,15,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,4,1,2,15,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,4,1,4,15,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,4,1,16,15,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,4,1,32,15,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,4,1,64,15,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,4,1,2,15,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,4,1,1,15,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,4,1,4,15,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,4,1,8,15,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,4,1,16,15,0.010714666297038397
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,4,1,32,15,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,4,1,64,15,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,4,1,1,31,0.011077333241701126
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,4,1,2,31,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,4,1,4,31,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,4,1,8,31,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,4,1,16,31,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,4,1,32,31,0.010202666744589806
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,4,1,64,31,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,4,1,1,31,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,4,1,2,31,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,4,1,16,31,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,4,1,8,31,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,4,1,4,31,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,4,1,32,31,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,4,1,64,31,0.010565333068370819
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,4,1,1,63,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,4,1,2,63,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,4,1,4,63,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,4,1,8,63,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,4,1,32,63,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,4,1,16,63,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,4,1,64,63,0.010565333068370819
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,4,1,1,63,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,4,1,2,63,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,4,1,4,63,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,4,1,16,63,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,4,1,8,63,0.010559999694426855
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,4,1,32,63,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,4,1,64,63,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,4,1,2,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,4,1,1,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,4,1,4,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,4,1,8,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,4,1,16,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,4,1,32,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,4,1,64,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,4,1,1,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,4,1,2,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,4,1,4,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,4,1,8,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,4,1,32,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,4,1,16,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,4,1,64,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,4,1,1,255,0.024405332903067272
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,4,1,2,255,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,4,1,4,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,4,1,8,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,4,1,16,255,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,4,1,32,255,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,4,1,64,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,4,1,2,255,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,4,1,1,255,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,4,1,4,255,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,4,1,8,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,4,1,16,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,4,1,32,255,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,4,1,64,255,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,4,1,1,511,0.04437333345413208
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,4,1,2,511,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,4,1,8,511,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,4,1,16,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,4,1,32,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,4,1,64,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,4,1,1,511,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,4,1,4,511,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,4,1,2,511,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,4,1,4,511,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,4,1,8,511,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,4,1,16,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,4,1,32,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,4,1,64,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,4,1,1,1023,0.08413867155710857
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,4,1,8,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,4,1,4,1023,0.08396800359090169
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,4,1,32,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,4,1,64,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,4,1,16,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,4,1,2,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,4,1,1,1023,0.08396800359090169
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,4,1,2,1023,0.08396800359090169
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,4,1,16,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,4,1,8,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,4,1,4,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,4,1,64,1023,0.08345599969228108
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,4,1,32,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,4,1,2,2047,0.16332800189654031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,4,1,1,2047,0.1634986698627472
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,4,1,8,2047,0.16315733393033346
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,4,1,16,2047,0.16315733393033346
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,4,1,4,2047,0.16315733393033346
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,4,1,32,2047,0.1629866659641266
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,4,1,64,2047,0.1629866659641266
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,4,1,1,2047,0.1634986698627472
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,4,1,8,2047,0.16315733393033346
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,4,1,32,2047,0.1629866659641266
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,4,1,16,2047,0.16332800189654031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,4,1,64,2047,0.16315733393033346
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,4,1,2,2047,0.16332800189654031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,4,1,2,4095,0.32238932450612384
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,4,1,4,2047,0.16332800189654031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,4,1,1,4095,0.32238932450612384
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,4,1,16,4095,0.3222186764081319
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,4,1,8,4095,0.3222186764081319
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,4,1,64,4095,0.3218773404757182
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,4,1,32,4095,0.3218773404757182
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,4,1,4,4095,0.3222186764081319
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,4,1,1,4095,0.32255999247233075
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,4,1,4,4095,0.3222186764081319
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,4,1,8,4095,0.32255999247233075
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,4,1,2,4095,0.32238932450612384
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,4,1,32,4095,0.32204800844192505
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,4,1,16,4095,0.32204800844192505
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,4,1,64,4095,0.32204800844192505
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,4,1,1,8191,0.6406826575597128
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,4,1,4,8191,0.640341321627299
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,4,1,2,8191,0.6401706536610922
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,4,1,8,8191,0.6408533255259196
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,4,1,64,8191,0.6399999856948853
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,4,1,32,8191,0.6398293177286783
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,4,1,16,8191,0.6399999856948853
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,4,1,1,8191,0.6405119895935059
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,4,1,4,8191,0.640341321627299
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,4,1,16,8191,0.6408533255259196
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,4,1,2,8191,0.640341321627299
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,4,1,8,8191,0.6401706536610922
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,4,1,32,8191,0.6401706536610922
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,4,1,64,8191,0.6399999856948853
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,4,1,2,16383,1.2844373385111492
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,4,1,1,16383,1.2764159838358562
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,4,1,8,16383,1.277781327565511
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,4,1,4,16383,1.276245355606079
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,4,1,32,16383,1.2759040196736653
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,4,1,16,16383,1.2760746479034424
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,4,1,1,16383,1.276586691538493
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,4,1,64,16383,1.2764159838358562
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,4,1,2,16383,1.276245355606079
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,4,1,32,16383,1.2760746479034424
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,4,1,8,16383,1.2760746479034424
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,4,1,16,16383,1.2776106993357341
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,4,1,4,16383,1.2764159838358562
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,4,1,64,16383,1.2760746479034424
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,4,1,1,32767,2.6082986195882163
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,4,1,2,32767,2.596522649129232
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,4,1,4,32767,2.5970346132914224
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,4,1,16,32767,2.5951573053995767
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,4,1,8,32767,2.597205320994059
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,4,1,32,32767,2.5978879928588867
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,4,1,64,32767,2.5941333770751953
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,4,1,1,32767,2.6050559679667153
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,4,1,8,32767,2.5956692695617676
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,4,1,4,32767,2.5941333770751953
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,4,1,2,32767,2.601301352183024
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,4,1,16,32767,2.59549872080485
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,4,1,32,32767,2.5961813926696777
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,4,1,1,65535,5.204479853312175
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,4,1,64,32767,2.5985706647237143
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,4,1,4,65535,5.189461390177409
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,4,1,2,65535,5.1923627853393555
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,4,1,16,65535,5.188437461853027
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,4,1,8,65535,5.182976086934407
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,4,1,64,65535,5.186389287312825
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,4,1,32,65535,5.1862185796101885
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,4,1,2,65535,5.203968048095703
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,4,1,1,65535,5.204821268717448
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,4,1,4,65535,5.188778559366862
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,4,1,8,65535,5.185023943583171
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,4,1,16,65535,5.189461390177409
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,4,1,64,65535,5.186047871907552
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,4,1,32,65535,5.182463963826497
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,4,1,1,131071,10.403669357299805
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,4,1,4,131071,10.376362482706705
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,4,1,2,131071,10.400938669840494
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,4,1,8,131071,10.373631795247396
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,4,1,16,131071,10.36680539449056
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,4,1,32,131071,10.370389302571615
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,4,1,64,131071,10.360832214355469
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,4,1,1,131071,10.406400044759115
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,4,1,2,131071,10.386602401733398
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,4,1,4,131071,10.376021067301432
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,4,1,8,131071,10.3819948832194
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,4,1,16,131071,10.372949600219727
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,4,1,32,131071,10.364416122436523
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,8,1,1,1,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,8,1,4,1,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,8,1,8,1,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,4,1,64,131071,10.368682861328125
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,8,1,32,1,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,8,1,16,1,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,8,1,64,1,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,8,1,2,1,0.010543999572594961
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,8,1,1,1,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,8,1,2,1,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,8,1,4,1,0.010570666442314783
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,8,1,32,1,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,8,1,16,1,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,8,1,8,1,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,8,1,64,1,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,8,1,1,3,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,8,1,2,3,0.01090666651725769
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,8,1,8,3,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,8,1,4,3,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,8,1,16,3,0.010399999717871347
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,8,1,32,3,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,8,1,2,3,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,8,1,1,3,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,8,1,64,3,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,8,1,16,3,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,8,1,8,3,0.021162666380405426
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,8,1,32,3,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,8,1,4,3,0.010869332899649939
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,8,1,64,3,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,8,1,1,7,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,8,1,2,7,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,8,1,4,7,0.010703999549150467
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,8,1,16,7,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,8,1,8,7,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,8,1,32,7,0.010527999450763067
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,8,1,64,7,0.019445333629846573
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,8,1,1,7,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,8,1,2,7,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,8,1,8,7,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,8,1,4,7,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,8,1,32,7,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,8,1,64,7,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,8,1,1,15,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,8,1,16,7,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,8,1,2,15,0.010575999816258749
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,8,1,4,15,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,8,1,8,15,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,8,1,16,15,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,8,1,32,15,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,8,1,64,15,0.010362666721145311
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,8,1,2,15,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,8,1,1,15,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,8,1,4,15,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,8,1,8,15,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,8,1,16,15,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,8,1,64,15,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,8,1,32,15,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,8,1,2,31,0.010725333044926325
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,8,1,1,31,0.01091733326514562
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,8,1,8,31,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,8,1,4,31,0.010522666076819101
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,8,1,16,31,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,8,1,32,31,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,8,1,1,31,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,8,1,4,31,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,8,1,64,31,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,8,1,2,31,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,8,1,16,31,0.011077333241701126
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,8,1,32,31,0.010399999717871347
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,8,1,64,31,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,8,1,1,63,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,8,1,2,63,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,8,1,4,63,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,8,1,8,31,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,8,1,16,63,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,8,1,8,63,0.01089599976936976
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,8,1,32,63,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,8,1,1,63,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,8,1,64,63,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,8,1,2,63,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,8,1,4,63,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,8,1,8,63,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,8,1,16,63,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,8,1,64,63,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,8,1,32,63,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,8,1,1,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,8,1,2,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,8,1,4,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,8,1,8,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,8,1,16,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,8,1,32,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,8,1,1,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,8,1,64,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,8,1,2,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,8,1,8,127,0.017407999684413273
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,8,1,4,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,8,1,16,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,8,1,32,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,8,1,64,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,8,1,1,255,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,8,1,2,255,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,8,1,8,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,8,1,4,255,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,8,1,16,255,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,8,1,32,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,8,1,1,255,0.024405332903067272
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,8,1,64,255,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,8,1,2,255,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,8,1,4,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,8,1,8,255,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,8,1,16,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,8,1,32,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,8,1,1,511,0.04437333345413208
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,8,1,64,255,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,8,1,4,511,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,8,1,2,511,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,8,1,16,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,8,1,8,511,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,8,1,32,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,8,1,2,511,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,8,1,64,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,8,1,1,511,0.04437333345413208
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,8,1,4,511,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,8,1,16,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,8,1,32,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,8,1,8,511,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,8,1,2,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,8,1,64,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,8,1,1,1023,0.08396800359090169
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,8,1,4,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,8,1,16,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,8,1,8,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,8,1,64,1023,0.08345599969228108
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,8,1,32,1023,0.08345599969228108
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,8,1,1,1023,0.08396800359090169
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,8,1,2,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,8,1,4,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,8,1,8,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,8,1,16,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,8,1,32,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,8,1,64,1023,0.08345599969228108
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,8,1,1,2047,0.16366933782895407
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,8,1,2,2047,0.16332800189654031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,8,1,4,2047,0.16332800189654031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,8,1,8,2047,0.16315733393033346
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,8,1,16,2047,0.1629866659641266
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,8,1,32,2047,0.16332800189654031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,8,1,1,2047,0.1634986698627472
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,8,1,64,2047,0.16315733393033346
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,8,1,4,2047,0.16332800189654031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,8,1,2,2047,0.1634986698627472
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,8,1,8,2047,0.16315733393033346
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,8,1,16,2047,0.16315733393033346
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,8,1,32,2047,0.1629866659641266
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,8,1,64,2047,0.16332800189654031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,8,1,1,4095,0.32238932450612384
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,8,1,2,4095,0.32238932450612384
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,8,1,4,4095,0.32204800844192505
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,8,1,8,4095,0.3222186764081319
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,8,1,16,4095,0.32204800844192505
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,8,1,32,4095,0.3218773404757182
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,8,1,64,4095,0.32204800844192505
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,8,1,1,4095,0.3232426643371582
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,8,1,2,4095,0.32238932450612384
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,8,1,4,4095,0.3222186764081319
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,8,1,32,4095,0.32204800844192505
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,8,1,8,4095,0.3222186764081319
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,8,1,64,4095,0.3218773404757182
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,8,1,16,4095,0.3218773404757182
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,8,1,2,8191,0.6413653294245402
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,8,1,1,8191,0.6406826575597128
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,8,1,4,8191,0.6401706536610922
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,8,1,8,8191,0.6401706536610922
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,8,1,16,8191,0.6398293177286783
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,8,1,64,8191,0.6399999856948853
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,8,1,1,8191,0.6405119895935059
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,8,1,2,8191,0.6413653294245402
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,8,1,4,8191,0.6405119895935059
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,8,1,8,8191,0.6401706536610922
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,8,1,16,8191,0.6399999856948853
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,8,1,32,8191,0.6399999856948853
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,8,1,32,8191,0.6399999856948853
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,8,1,64,8191,0.6399999856948853
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,8,1,1,16383,1.3093547026316326
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,8,1,2,16383,1.3064533074696858
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,8,1,4,16383,1.302186648050944
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,8,1,8,16383,1.3110613028208415
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,8,1,16,16383,1.3003093401590984
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,8,1,64,16383,1.3015039761861165
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,8,1,1,16383,1.3067946434020996
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,8,1,2,16383,1.3054292996724446
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,8,1,4,16383,1.3061119715372722
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,8,1,8,16383,1.3015039761861165
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,8,1,16,16383,1.302186648050944
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,8,1,32,16383,1.303381363550822
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,8,1,32,16383,1.3013333479563396
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,8,1,64,16383,1.3009920120239258
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,8,1,1,32767,2.6072746912638345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,8,1,2,32767,2.6050559679667153
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,8,1,4,32767,2.605397383371989
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,8,1,8,32767,2.596522649129232
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,8,1,16,32767,2.5983999570210776
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,8,1,64,32767,2.5970346132914224
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,8,1,1,32767,2.6082986195882163
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,8,1,2,32767,2.601130644480387
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,8,1,4,32767,2.6007893880208335
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,8,1,8,32767,2.6031786600748696
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,8,1,32,32767,2.6014720598856607
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,8,1,16,32767,2.596010684967041
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,8,1,32,32767,2.598741372426351
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,8,1,64,32767,2.5968640645345054
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,8,1,1,65535,5.214378674825032
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,8,1,2,65535,5.198506673177083
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,8,1,4,65535,5.197141329447429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,8,1,8,65535,5.20089594523112
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,8,1,32,65535,5.1877546310424805
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,8,1,64,65535,5.192192077636719
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,8,1,1,65535,5.207893371582031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,8,1,16,65535,5.195093472798665
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,8,1,2,65535,5.202261288960774
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,8,1,4,65535,5.195093472798665
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,8,1,8,65535,5.193728129069011
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,8,1,16,65535,5.201578776041667
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,8,1,32,65535,5.188096046447754
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,8,1,64,65535,5.191679954528809
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,8,1,1,131071,10.406911849975586
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,8,1,4,131071,10.384213129679361
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,8,1,8,131071,10.375680287679037
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,8,1,16,131071,10.376533508300781
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,8,1,32,131071,10.39018694559733
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,8,1,2,131071,10.401962916056315
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,8,1,64,131071,10.371072133382162
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,8,1,1,131071,10.41971206665039
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,8,1,2,131071,10.389504114786783
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,8,1,4,131071,10.384725570678711
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,16,1,1,1,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,8,1,8,131071,10.377386728922525
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,8,1,16,131071,10.377045313517252
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,8,1,32,131071,10.391722361246744
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,16,1,2,1,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,16,1,4,1,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,16,1,8,1,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,8,1,64,131071,10.38267707824707
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,16,1,16,1,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,16,1,32,1,0.0103946669648091
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,16,1,64,1,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,16,1,2,1,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,16,1,1,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,16,1,4,1,0.010559999694426855
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,16,1,8,1,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,16,1,64,1,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,16,1,32,1,0.010351999973257383
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,16,1,1,3,0.010885333021481832
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,16,1,2,3,0.010746666540702185
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,16,1,4,3,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,16,1,16,1,0.013936000565687815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,16,1,16,3,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,16,1,8,3,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,16,1,64,3,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,16,1,1,3,0.011231999844312668
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,16,1,32,3,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,16,1,2,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,16,1,4,3,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,16,1,16,3,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,16,1,32,3,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,16,1,64,3,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,16,1,8,3,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,16,1,1,7,0.010879999647537867
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,16,1,4,7,0.010378666842977205
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,16,1,2,7,0.013173333058754602
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,16,1,8,7,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,16,1,32,7,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,16,1,16,7,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,16,1,64,7,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,16,1,4,7,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,16,1,16,7,0.011247999966144562
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,16,1,2,7,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,16,1,32,7,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,16,1,8,7,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,16,1,64,7,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,16,1,1,7,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,16,1,1,15,0.011055999745925268
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,16,1,2,15,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,16,1,4,15,0.011946666985750198
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,16,1,8,15,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,16,1,16,15,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,16,1,32,15,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,16,1,64,15,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,16,1,2,15,0.011071999867757162
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,16,1,4,15,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,16,1,8,15,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,16,1,16,15,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,16,1,1,15,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,16,1,32,15,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,16,1,1,31,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,16,1,64,15,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,16,1,2,31,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,16,1,8,31,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,16,1,16,31,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,16,1,4,31,0.011247999966144562
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,16,1,64,31,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,16,1,1,31,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,16,1,2,31,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,16,1,4,31,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,16,1,32,31,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,16,1,8,31,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,16,1,16,31,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,16,1,32,31,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,16,1,1,63,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,16,1,64,31,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,16,1,4,63,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,16,1,2,63,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,16,1,16,63,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,16,1,32,63,0.0103946669648091
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,16,1,8,63,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,16,1,64,63,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,16,1,1,63,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,16,1,2,63,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,16,1,4,63,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,16,1,8,63,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,16,1,32,63,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,16,1,16,63,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,16,1,2,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,16,1,1,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,16,1,4,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,16,1,8,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,16,1,64,63,0.0103946669648091
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,16,1,16,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,16,1,32,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,16,1,64,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,16,1,1,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,16,1,4,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,16,1,2,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,16,1,8,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,16,1,32,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,16,1,64,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,16,1,1,255,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,16,1,2,255,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,16,1,16,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,16,1,4,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,16,1,8,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,16,1,16,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,16,1,32,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,16,1,64,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,16,1,1,255,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,16,1,2,255,0.024405332903067272
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,16,1,16,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,16,1,8,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,16,1,32,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,16,1,64,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,16,1,4,255,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,16,1,1,511,0.04437333345413208
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,16,1,2,511,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,16,1,4,511,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,16,1,8,511,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,16,1,16,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,16,1,64,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,16,1,2,511,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,16,1,4,511,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,16,1,8,511,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,16,1,16,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,16,1,1,511,0.04437333345413208
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,16,1,32,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,16,1,32,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,16,1,1,1023,0.08413867155710857
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,16,1,2,1023,0.08413867155710857
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,16,1,64,511,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,16,1,4,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,16,1,8,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,16,1,32,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,16,1,16,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,16,1,8,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,16,1,1,1023,0.08430932958920796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,16,1,64,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,16,1,2,1023,0.08396800359090169
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,16,1,4,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,16,1,16,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,16,1,32,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,16,1,4,2047,0.1634986698627472
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,16,1,1,2047,0.16383999586105347
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,16,1,64,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,16,1,2,2047,0.1634986698627472
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,16,1,16,2047,0.1629866659641266
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,16,1,8,2047,0.16366933782895407
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,16,1,32,2047,0.1629866659641266
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,16,1,64,2047,0.16315733393033346
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,16,1,1,2047,0.16383999586105347
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,16,1,2,2047,0.16332800189654031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,16,1,4,2047,0.16332800189654031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,16,1,16,2047,0.16315733393033346
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,16,1,32,2047,0.1634986698627472
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,16,1,64,2047,0.1634986698627472
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,16,1,1,4095,0.3546453317006429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,16,1,2,4095,0.32255999247233075
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,16,1,4,4095,0.32238932450612384
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,16,1,8,2047,0.16332800189654031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,16,1,32,4095,0.32290132840474445
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,16,1,8,4095,0.32238932450612384
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,16,1,16,4095,0.32204800844192505
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,16,1,64,4095,0.32204800844192505
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,16,1,1,4095,0.3227306604385376
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,16,1,4,4095,0.32238932450612384
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,16,1,2,4095,0.32238932450612384
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,16,1,8,4095,0.3222186764081319
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,16,1,16,4095,0.3227306604385376
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,16,1,64,4095,0.32204800844192505
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,16,1,32,4095,0.3222186764081319
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,16,1,2,8191,0.6597973505655924
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,16,1,1,8191,0.6650880177815756
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,16,1,4,8191,0.659114678700765
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,16,1,16,8191,0.6594560146331787
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,16,1,8,8191,0.6592853466669718
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,16,1,32,8191,0.6584320068359375
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,16,1,64,8191,0.65774933497111
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,16,1,2,8191,0.6592853466669718
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,16,1,4,8191,0.6599680185317993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,16,1,8,8191,0.6586026748021444
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,16,1,1,8191,0.6661119858423868
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,16,1,16,8191,0.6594560146331787
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,16,1,32,8191,0.659114678700765
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,16,1,64,8191,0.65774933497111
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,16,1,1,16383,1.3172053496042888
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,16,1,2,16383,1.3114026387532551
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,16,1,4,16383,1.330346663792928
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,16,1,8,16383,1.3096960385640461
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,16,1,32,16383,1.311743974685669
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,16,1,64,16383,1.3108906745910645
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,16,1,2,16383,1.3124266465504963
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,16,1,16,16383,1.311743974685669
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,16,1,1,16383,1.3218133449554443
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,16,1,4,16383,1.308672030766805
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,16,1,8,16383,1.3108906745910645
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,16,1,16,16383,1.3098666667938232
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,16,1,32,16383,1.311743974685669
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,16,1,64,16383,1.3110613028208415
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,16,1,1,32767,2.6289493242899575
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,16,1,4,32767,2.6197333335876465
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,16,1,8,32767,2.609834671020508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,16,1,16,32767,2.614954630533854
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,16,1,2,32767,2.625194708506266
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,16,1,32,32767,2.611541271209717
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,16,1,64,32767,2.6163199742635093
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,16,1,1,32767,2.635434627532959
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,16,1,2,32767,2.618879954020182
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,16,1,4,32767,2.6192213694254556
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,16,1,8,32767,2.609663963317871
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,16,1,16,32767,2.6152960459391275
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,16,1,64,32767,2.6112000147501626
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,16,1,32,32767,2.6183679898579917
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,16,1,1,65535,5.258069356282552
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,16,1,2,65535,5.234005292256673
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,16,1,4,65535,5.215573310852051
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,16,1,16,65535,5.217962582906087
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,16,1,8,65535,5.221376101175944
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,16,1,32,65535,5.216767946879069
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,16,1,64,65535,5.213525454203288
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,16,1,2,65535,5.231786727905273
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,16,1,4,65535,5.221205393473308
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,16,1,1,65535,5.2616532643636065
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,16,1,8,65535,5.210282643636067
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,16,1,32,65535,5.220863978068034
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,16,1,64,65535,5.217621485392253
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,16,1,16,65535,5.207893371582031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,16,1,1,131071,10.484394709269205
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,16,1,2,131071,10.449237187703451
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,16,1,4,131071,10.444458643595377
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,16,1,16,131071,10.414080301920572
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,16,1,8,131071,10.42961057027181
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,16,1,32,131071,10.433877309163412
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,16,1,1,131071,10.456746419270834
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,16,1,64,131071,10.426709493001303
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,16,1,2,131071,10.44701894124349
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,16,1,4,131071,10.434730529785156
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,32,1,1,1,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,32,1,2,1,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,16,1,16,131071,10.412714640299479
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,16,1,8,131071,10.434218724568685
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,32,1,4,1,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,16,1,32,131071,10.431488037109375
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,16,1,64,131071,10.430293401082357
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,32,1,8,1,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,32,1,16,1,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,32,1,32,1,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,32,1,2,1,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,32,1,1,1,0.011946666985750198
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,32,1,64,1,0.010714666297038397
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,32,1,4,1,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,32,1,16,1,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,32,1,8,1,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,32,1,64,1,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,32,1,32,1,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,32,1,2,3,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,32,1,1,3,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,32,1,4,3,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,32,1,8,3,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,32,1,16,3,0.010543999572594961
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,32,1,32,3,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,32,1,1,3,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,32,1,64,3,0.0363520011305809
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,32,1,2,3,0.010741333166758219
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,32,1,4,3,0.012437333663304647
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,32,1,8,3,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,32,1,16,3,0.010538666198650995
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,32,1,32,3,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,32,1,1,7,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,32,1,2,7,0.012282667060693106
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,32,1,64,3,0.017237332959969837
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,32,1,4,7,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,32,1,16,7,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,32,1,8,7,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,32,1,32,7,0.010570666442314783
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,32,1,64,7,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,32,1,1,7,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,32,1,2,7,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,32,1,8,7,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,32,1,16,7,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,32,1,32,7,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,32,1,4,7,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,32,1,64,7,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,32,1,1,15,0.01156266654531161
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,32,1,2,15,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,32,1,4,15,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,32,1,8,15,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,32,1,16,15,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,32,1,32,15,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,32,1,2,15,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,32,1,1,15,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,32,1,64,15,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,32,1,4,15,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,32,1,8,15,0.010709332923094431
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,32,1,16,15,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,32,1,32,15,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,32,1,64,15,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,32,1,1,31,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,32,1,2,31,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,32,1,16,31,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,32,1,8,31,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,32,1,4,31,0.01110400011142095
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,32,1,32,31,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,32,1,1,31,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,32,1,2,31,0.011066666493813196
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,32,1,64,31,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,32,1,4,31,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,32,1,8,31,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,32,1,16,31,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,32,1,2,63,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,32,1,64,31,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,32,1,1,63,0.011733333269755045
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,32,1,8,63,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,32,1,32,31,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,32,1,4,63,0.01091733326514562
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,32,1,32,63,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,32,1,16,63,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,32,1,64,63,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,32,1,1,63,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,32,1,4,63,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,32,1,2,63,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,32,1,16,63,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,32,1,8,63,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,32,1,64,63,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,32,1,32,63,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,32,1,1,127,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,32,1,2,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,32,1,4,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,32,1,8,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,32,1,16,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,32,1,64,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,32,1,1,127,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,32,1,2,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,32,1,4,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,32,1,8,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,32,1,32,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,32,1,32,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,32,1,16,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,32,1,64,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,32,1,1,255,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,32,1,2,255,0.024405332903067272
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,32,1,4,255,0.024405332903067272
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,32,1,8,255,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,32,1,32,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,32,1,64,255,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,32,1,1,255,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,32,1,2,255,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,32,1,16,255,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,32,1,8,255,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,32,1,4,255,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,32,1,16,255,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,32,1,32,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,32,1,64,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,32,1,1,511,0.04488533238569895
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,32,1,2,511,0.04437333345413208
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,32,1,8,511,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,32,1,16,511,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,32,1,32,511,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,32,1,64,511,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,32,1,4,511,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,32,1,1,511,0.04505600035190582
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,32,1,2,511,0.04437333345413208
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,32,1,4,511,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,32,1,8,511,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,32,1,32,511,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,32,1,16,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,32,1,64,511,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,32,1,4,1023,0.08413867155710857
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,32,1,2,1023,0.08430932958920796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,32,1,8,1023,0.08396800359090169
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,32,1,16,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,32,1,1,1023,0.08482133348782857
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,32,1,64,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,32,1,32,1023,0.08396800359090169
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,32,1,1,1023,0.08482133348782857
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,32,1,2,1023,0.08413867155710857
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,32,1,4,1023,0.08396800359090169
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,32,1,8,1023,0.08396800359090169
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,32,1,16,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,32,1,64,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,32,1,1,2047,0.16486400365829468
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,32,1,2,2047,0.16366933782895407
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,32,1,4,2047,0.16366933782895407
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,32,1,32,1023,0.08396800359090169
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,32,1,8,2047,0.1634986698627472
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,32,1,16,2047,0.16332800189654031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,32,1,32,2047,0.16315733393033346
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,32,1,64,2047,0.16332800189654031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,32,1,1,2047,0.1646933356920878
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,32,1,2,2047,0.16383999586105347
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,32,1,16,2047,0.16332800189654031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,32,1,4,2047,0.1634986698627472
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,32,1,8,2047,0.1634986698627472
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,32,1,32,2047,0.16332800189654031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,32,1,64,2047,0.16332800189654031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,32,1,1,4095,0.3572053511937459
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,32,1,2,4095,0.3537919918696086
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,32,1,4,4095,0.3510613441467285
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,32,1,8,4095,0.34884266058603924
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,32,1,16,4095,0.3495253324508667
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,32,1,32,4095,0.3500373363494873
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,32,1,64,4095,0.3485013246536255
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,32,1,2,4095,0.3546453317006429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,32,1,1,4095,0.3580586512883504
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,32,1,8,4095,0.35089067618052167
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,32,1,16,4095,0.34884266058603924
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,32,1,4,4095,0.35089067618052167
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,32,1,32,4095,0.3500373363494873
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,32,1,64,4095,0.35089067618052167
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,32,1,1,8191,0.6959786415100098
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,32,1,2,8191,0.6941013336181641
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,32,1,4,8191,0.6912000179290771
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,32,1,16,8191,0.6877866586049398
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,32,1,64,8191,0.6901760101318359
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,32,1,8,8191,0.6893226305643717
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,32,1,32,8191,0.6871039867401123
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,32,1,2,8191,0.6923946539560953
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,32,1,4,8191,0.6910293102264404
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,32,1,1,8191,0.6992213726043701
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,32,1,8,8191,0.6910293102264404
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,32,1,16,8191,0.6877866586049398
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,32,1,64,8191,0.6898346741994222
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,32,1,1,16383,1.3743786811828613
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,32,1,32,8191,0.6891520023345947
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,32,1,2,16383,1.3636266390482585
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,32,1,4,16383,1.36516269048055
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,32,1,16,16383,1.3595306078592937
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,32,1,8,16383,1.362943967183431
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,32,1,32,16383,1.354581356048584
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,32,1,64,16383,1.3574825922648113
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,32,1,1,16383,1.3764266967773438
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,32,1,2,16383,1.366869290669759
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,32,1,4,16383,1.3591893513997395
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,32,1,8,16383,1.3666987419128418
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,32,1,16,16383,1.3603839874267578
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,32,1,32,16383,1.3632853825887044
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,32,1,64,16383,1.3564586639404297
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,32,1,1,32767,2.7281068166097007
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,32,1,2,32767,2.7105280558268228
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,32,1,4,32767,2.709845225016276
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,32,1,16,32767,2.7195733388264975
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,32,1,8,32767,2.7054080963134766
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,32,1,32,32767,2.7088212966918945
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,32,1,2,32767,2.713088035583496
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,32,1,1,32767,2.7185494105021157
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,32,1,4,32767,2.703189214070638
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,32,1,16,32767,2.7149651845296225
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,32,1,8,32767,2.7088212966918945
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,32,1,64,32767,2.7149651845296225
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,32,1,32,32767,2.705749193827311
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,32,1,64,32767,2.7055788040161133
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,32,1,1,65535,5.4493865966796875
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,32,1,2,65535,5.401941299438477
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,32,1,4,65535,5.377194722493489
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,32,1,64,65535,5.3850453694661455
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,32,1,16,65535,5.401429494222005
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,32,1,1,65535,5.455018361409505
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,32,1,8,65535,5.400234858194987
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,32,1,32,65535,5.407914479573567
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,32,1,2,65535,5.41047477722168
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,32,1,4,65535,5.381631851196289
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,32,1,8,65535,5.37002690633138
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,32,1,16,65535,5.389312108357747
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,32,1,32,65535,5.39784558614095
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,32,1,64,65535,5.405525207519531
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,32,1,1,131071,10.89467748006185
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,32,1,4,131071,10.78442637125651
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,32,1,8,131071,10.718037923177084
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,32,1,2,131071,10.818901062011719
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,32,1,16,131071,10.740394592285156
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,32,1,32,131071,10.75814437866211
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,32,1,64,131071,10.759167989095053
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,32,1,1,131071,10.89621353149414
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,32,1,2,131071,10.81548817952474
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,64,1,1,1,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,32,1,4,131071,10.77401606241862
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,32,1,8,131071,10.729301452636719
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,64,1,2,1,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,32,1,16,131071,10.792619069417318
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,64,1,8,1,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,32,1,32,131071,10.715989430745443
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,64,1,4,1,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,32,1,64,131071,10.758655548095703
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,64,1,32,1,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,64,1,16,1,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,64,1,64,1,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,64,1,2,1,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,64,1,1,1,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,64,1,4,1,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,64,1,16,1,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,64,1,64,1,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,64,1,32,1,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,64,1,8,1,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,64,1,1,3,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,64,1,2,3,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,64,1,4,3,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,64,1,8,3,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,64,1,16,3,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,64,1,32,3,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,64,1,64,3,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,64,1,1,3,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,64,1,4,3,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,64,1,2,3,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,64,1,8,3,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,64,1,16,3,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,64,1,32,3,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,64,1,64,3,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,64,1,1,7,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,64,1,2,7,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,64,1,4,7,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,64,1,16,7,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,64,1,32,7,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,64,1,8,7,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,64,1,1,7,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,64,1,64,7,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,64,1,4,7,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,64,1,2,7,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,64,1,8,7,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,64,1,16,7,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,64,1,64,7,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,64,1,32,7,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,64,1,1,15,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,64,1,2,15,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,64,1,4,15,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,64,1,8,15,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,64,1,16,15,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,64,1,32,15,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,64,1,64,15,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,64,1,1,15,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,64,1,4,15,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,64,1,2,15,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,64,1,8,15,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,64,1,16,15,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,64,1,32,15,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,64,1,1,31,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,64,1,64,15,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,64,1,4,31,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,64,1,2,31,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,64,1,16,31,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,64,1,8,31,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,64,1,32,31,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,64,1,64,31,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,64,1,2,31,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,64,1,1,31,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,64,1,4,31,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,64,1,8,31,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,64,1,32,31,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,64,1,1,63,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,64,1,64,31,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,64,1,16,31,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,64,1,4,63,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,64,1,8,63,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,64,1,2,63,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,64,1,16,63,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,64,1,32,63,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,64,1,64,63,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,64,1,2,63,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,64,1,1,63,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,64,1,4,63,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,64,1,8,63,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,64,1,16,63,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,64,1,32,63,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,64,1,64,63,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,64,1,1,127,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,64,1,4,127,0.021333334346612293
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,64,1,8,127,0.021162666380405426
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,64,1,2,127,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,64,1,64,127,0.020992000897725422
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,64,1,16,127,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,64,1,32,127,0.020992000897725422
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,64,1,1,127,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,64,1,2,127,0.021503999829292297
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,64,1,4,127,0.021333334346612293
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,64,1,16,127,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,64,1,1,255,0.037717332442601524
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,64,1,32,127,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,64,1,8,127,0.021162666380405426
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,64,1,64,127,0.020992000897725422
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,64,1,2,255,0.03737599899371465
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,64,1,4,255,0.037205333511034645
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,64,1,8,255,0.036864000062147774
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,64,1,16,255,0.036864000062147774
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,64,1,1,255,0.03788800040880839
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,64,1,2,255,0.03754666695992152
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,64,1,32,255,0.03669333209594091
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,64,1,8,255,0.03703466554482778
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,64,1,4,255,0.037205333511034645
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,64,1,64,255,0.03669333209594091
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,64,1,16,255,0.03669333209594091
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,64,1,32,255,0.036864000062147774
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,64,1,64,255,0.03669333209594091
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,64,1,2,511,0.06963199873765309
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,64,1,1,511,0.06946133573849995
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,64,1,4,511,0.06894933183987935
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,64,1,32,511,0.06843733290831248
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,64,1,16,511,0.0682666649421056
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,64,1,8,511,0.06843733290831248
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,64,1,64,511,0.06877866884072621
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,64,1,1,511,0.06946133573849995
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,64,1,4,511,0.06929066777229309
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,64,1,2,511,0.06946133573849995
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,64,1,8,511,0.06860800087451935
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,64,1,32,511,0.0682666649421056
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,64,1,16,511,0.06843733290831248
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,64,1,64,511,0.06860800087451935
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,64,1,2,1023,0.13346133629480997
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,64,1,1,1023,0.13499733805656433
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,64,1,4,1023,0.1327786644299825
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,64,1,8,1023,0.13192533453305563
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,64,1,32,1023,0.13141333063443503
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,64,1,16,1023,0.13158399860064188
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,64,1,1,1023,0.13397333025932312
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,64,1,64,1023,0.13141333063443503
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,64,1,2,1023,0.13363200426101685
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,64,1,4,1023,0.13260799646377563
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,64,1,8,1023,0.13243732849756876
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,64,1,16,1023,0.13158399860064188
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,64,1,32,1023,0.13158399860064188
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,64,1,64,1023,0.13243732849756876
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,64,1,1,2047,0.265557328859965
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,64,1,2,2047,0.2640213370323181
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,64,1,4,2047,0.26231465737024945
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,64,1,8,2047,0.26333866516749066
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,64,1,16,2047,0.2629973292350769
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,64,1,64,2047,0.2624853253364563
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,64,1,32,2047,0.26265599330266315
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,64,1,1,2047,0.2681173284848531
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,64,1,2,2047,0.26368000109990436
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,64,1,4,2047,0.26282666126887005
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,64,1,16,2047,0.26282666126887005
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,64,1,8,2047,0.2621440092722575
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,64,1,32,2047,0.26282666126887005
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,64,1,64,2047,0.26231465737024945
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,64,1,2,4095,0.5157546599706014
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,64,1,1,4095,0.5191680192947388
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,64,1,4,4095,0.5145599842071533
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,64,1,8,4095,0.5140479803085327
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,64,1,16,4095,0.5143893162409464
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,64,1,32,4095,0.5143893162409464
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,64,1,64,4095,0.5142186482747396
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,64,1,1,4095,0.5186560153961182
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,64,1,2,4095,0.5154133240381876
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,64,1,8,4095,0.5147306521733602
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,64,1,4,4095,0.5147306521733602
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,64,1,32,4095,0.514901320139567
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,64,1,16,4095,0.5143893162409464
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,64,1,64,4095,0.5137066841125488
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,64,1,1,8191,1.0393599669138591
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,64,1,2,8191,1.0362880229949951
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,64,1,4,8191,1.034069299697876
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,64,1,8,8191,1.0315093199412029
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,64,1,32,8191,1.0335573355356853
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,64,1,16,8191,1.031167984008789
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,64,1,64,8191,1.0287786324818928
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,64,1,2,8191,1.036629358927409
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,64,1,1,8191,1.0397013028462727
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,64,1,4,8191,1.0345813433329265
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,64,1,8,8191,1.032362699508667
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,64,1,16,8191,1.0320213635762532
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,64,1,64,8191,1.0315093199412029
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,64,1,32,8191,1.048746665318807
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,64,1,1,16383,2.182485262552897
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,64,1,2,16383,2.1761706670125327
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,64,1,8,16383,2.1678080558776855
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,64,1,4,16383,2.1666132609049478
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,64,1,16,16383,2.16320006052653
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,64,1,32,16383,2.1609813372294107
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,64,1,64,16383,2.1666132609049478
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,64,1,1,16383,2.197333335876465
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,64,1,2,16383,2.169856071472168
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,64,1,16,16383,2.1664427121480307
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,64,1,4,16383,2.168490727742513
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,64,1,8,16383,2.168831984202067
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,64,1,64,16383,2.163541316986084
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,64,1,32,16383,2.164053281148275
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,64,1,1,32767,4.520959854125977
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,64,1,4,32767,4.472319920857747
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,64,1,2,32767,4.492970784505208
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,64,1,8,32767,4.476586659749349
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,64,1,16,32767,4.476245244344075
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,64,1,32,32767,4.467029253641765
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,64,1,64,32767,4.466517448425293
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,64,1,1,32767,4.50815995534261
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,64,1,2,32767,4.481706619262695
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,64,1,4,32767,4.473685264587402
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,64,1,8,32767,4.479658762613933
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,64,1,16,32767,4.470954577128093
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,64,1,64,32767,4.474538803100586
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,64,1,32,32767,4.474538803100586
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,64,1,2,65535,9.105749130249023
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,64,1,1,65535,9.148415883382162
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,64,1,8,65535,9.062570571899414
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,64,1,4,65535,9.113429387410482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,64,1,16,65535,9.125888188680014
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,64,1,32,65535,9.05728022257487
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,64,1,1,65535,9.164800008138021
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,64,1,64,65535,9.085952123006185
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,64,1,2,65535,9.100970586140951
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,64,1,4,65535,9.138005574544271
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,64,1,16,65535,9.090389251708984
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,64,1,8,65535,9.044650395711264
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,64,1,32,65535,9.110528310139975
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,64,1,64,65535,9.085952123006185
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,64,1,1,131071,18.558123270670574
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,64,1,2,131071,18.47040049235026
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,64,1,4,131071,18.4268798828125
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,64,1,8,131071,18.362879435221355
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,64,1,16,131071,18.427562713623047
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,64,1,64,131071,18.358272552490234
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,64,1,1,131071,18.540543874104817
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,64,1,32,131071,18.437973022460938
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,128,1,1,1,0.016895999511082966
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,128,1,2,1,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,64,1,2,131071,18.493610382080078
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,128,1,4,1,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,128,1,8,1,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,64,1,4,131071,18.408789316813152
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,64,1,8,131071,18.449748992919922
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,64,1,16,131071,18.41629918416341
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,128,1,16,1,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,128,1,32,1,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,64,1,32,131071,18.390015920003254
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,64,1,64,131071,18.422613779703777
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,128,1,64,1,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,128,1,1,1,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,128,1,2,1,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,128,1,4,1,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,128,1,8,1,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,128,1,16,1,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,128,1,32,1,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,128,1,2,3,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,128,1,64,1,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,128,1,1,3,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,128,1,4,3,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,128,1,8,3,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,128,1,16,3,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,128,1,64,3,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,128,1,32,3,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,128,1,1,3,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,128,1,4,3,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,128,1,2,3,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,128,1,8,3,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,128,1,16,3,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,128,1,32,3,0.01634666696190834
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,128,1,64,3,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,128,1,1,7,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,128,1,2,7,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,128,1,4,7,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,128,1,8,7,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,128,1,16,7,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,128,1,32,7,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,128,1,64,7,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,128,1,1,7,0.016895999511082966
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,128,1,2,7,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,128,1,4,7,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,128,1,8,7,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,128,1,16,7,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,128,1,32,7,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,128,1,64,7,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,128,1,1,15,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,128,1,2,15,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,128,1,8,15,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,128,1,4,15,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,128,1,16,15,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,128,1,32,15,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,128,1,64,15,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,128,1,1,15,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,128,1,2,15,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,128,1,4,15,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,128,1,8,15,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,128,1,16,15,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,128,1,32,15,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,128,1,64,15,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,128,1,1,31,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,128,1,4,31,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,128,1,2,31,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,128,1,8,31,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,128,1,16,31,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,128,1,32,31,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,128,1,64,31,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,128,1,1,31,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,128,1,2,31,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,128,1,4,31,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,128,1,8,31,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,128,1,16,31,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,128,1,32,31,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,128,1,64,31,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,128,1,1,63,0.016895999511082966
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,128,1,2,63,0.015872000406185787
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,128,1,4,63,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,128,1,8,63,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,128,1,16,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,128,1,32,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,128,1,64,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,128,1,1,63,0.0170666662355264
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,128,1,2,63,0.015872000406185787
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,128,1,4,63,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,128,1,8,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,128,1,16,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,128,1,32,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,128,1,64,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,128,1,1,127,0.027818667391935985
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,128,1,2,127,0.025770666698614757
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,128,1,4,127,0.025429333249727886
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,128,1,8,127,0.024746666351954143
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,128,1,16,127,0.024746666351954143
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,128,1,32,127,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,128,1,64,127,0.025258667767047882
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,128,1,1,127,0.028501334289709728
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,128,1,2,127,0.025770666698614757
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,128,1,4,127,0.025258667767047882
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,128,1,8,127,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,128,1,16,127,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,128,1,32,127,0.024405332903067272
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,128,1,1,255,0.04898133377234141
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,128,1,64,127,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,128,1,2,255,0.04659200211366018
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,128,1,4,255,0.04454400142033895
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,128,1,8,255,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,128,1,16,255,0.04369066655635834
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,128,1,32,255,0.04471466441949209
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,128,1,64,255,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,128,1,1,255,0.04949333270390829
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,128,1,2,255,0.046762665112813316
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,128,1,4,255,0.045909335215886436
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,128,1,8,255,0.04539733131726583
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,128,1,16,255,0.043178667624791466
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,128,1,32,255,0.04437333345413208
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,128,1,1,511,0.09198932846387227
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,128,1,64,255,0.04437333345413208
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,128,1,2,511,0.0865280032157898
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,128,1,8,511,0.08567466338475545
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,128,1,4,511,0.08482133348782857
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,128,1,16,511,0.08516266942024231
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,128,1,64,511,0.08243200182914734
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,128,1,32,511,0.08345599969228108
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,128,1,1,511,0.0936959981918335
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,128,1,2,511,0.0865280032157898
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,128,1,4,511,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,128,1,8,511,0.08413867155710857
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,128,1,32,511,0.08243200182914734
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,128,1,64,511,0.08499200145403545
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,128,1,1,1023,0.25975465774536133
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,128,1,2,1023,0.2510506709416707
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,128,1,8,1023,0.24320000410079956
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,128,1,4,1023,0.24644267559051514
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,128,1,16,511,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,128,1,16,1023,0.2435413400332133
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,128,1,32,1023,0.24064000447591147
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,128,1,64,1023,0.24064000447591147
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,128,1,1,1023,0.25941334168116253
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,128,1,4,1023,0.2469546596209208
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,128,1,2,1023,0.25173334280649823
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,128,1,8,1023,0.244053324063619
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,128,1,32,1023,0.24166399240493774
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,128,1,64,1023,0.24132267634073892
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,128,1,1,2047,0.4805973370869954
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,128,1,2,2047,0.47325865427652997
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,128,1,16,1023,0.24183466037114462
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,128,1,4,2047,0.46830932299296063
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,128,1,8,2047,0.4666026830673218
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,128,1,16,2047,0.4628479878107707
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,128,1,32,2047,0.4628479878107707
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,128,1,64,2047,0.4618240197499593
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,128,1,1,2047,0.48076800505320233
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,128,1,2,2047,0.4739413261413574
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,128,1,16,2047,0.4638719956080119
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,128,1,8,2047,0.4657493432362874
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,128,1,64,2047,0.4621653159459432
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,128,1,1,4095,0.9216000239054362
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,128,1,32,2047,0.48298664887746173
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,128,1,2,4095,0.9154559771219889
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,128,1,4,2047,0.4689919948577881
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,128,1,4,4095,0.9093120098114014
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,128,1,8,4095,0.9060693581899008
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,128,1,32,4095,0.9026559988657633
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,128,1,16,4095,0.904533306757609
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,128,1,64,4095,0.9011200269063314
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,128,1,1,4095,0.9222826957702637
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,128,1,2,4095,0.9135786692301432
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,128,1,8,4095,0.904533306757609
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,128,1,16,4095,0.9033386707305908
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,128,1,32,4095,0.9053866863250732
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,128,1,64,4095,0.9052159786224365
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,128,1,4,4095,0.9105066458384196
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,128,1,1,8191,1.8107733726501465
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,128,1,2,8191,1.8001920382181804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,128,1,4,8191,1.7942186991373699
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,128,1,8,8191,1.7890987396240234
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,128,1,16,8191,1.78875732421875
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,128,1,32,8191,1.788416067759196
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,128,1,64,8191,1.7827839851379395
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,128,1,2,8191,1.7969493865966797
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,128,1,4,8191,1.7933653195699055
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,128,1,8,8191,1.790463924407959
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,128,1,32,8191,1.7935360272725422
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,128,1,1,8191,1.8102614084879558
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,128,1,16,8191,1.8121387163798015
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,128,1,64,8191,1.7919999758402507
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,128,1,1,16383,3.58024533589681
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,128,1,2,16383,3.5684693654378257
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,128,1,4,16383,3.5601065953572593
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,128,1,8,16383,3.559765179951986
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,128,1,16,16383,3.552255948384603
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,128,1,64,16383,3.5560105641682944
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,128,1,1,16383,3.5746132532755532
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,128,1,2,16383,3.567274729410807
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,128,1,32,16383,3.5392853418986
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,128,1,4,16383,3.559424082438151
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,128,1,8,16383,3.5520852406819663
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,128,1,16,16383,3.54474671681722
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,128,1,32,16383,3.548330624898275
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,128,1,64,16383,3.5681279500325522
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,128,1,1,32767,7.114410400390625
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,128,1,2,32767,7.085909525553386
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,128,1,8,32767,7.078229268391927
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,128,1,16,32767,7.070037206013997
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,128,1,32,32767,7.070720036824544
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,128,1,64,32767,7.064064025878906
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,128,1,4,32767,7.082154591878255
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,128,1,1,32767,7.109290440877278
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,128,1,2,32767,7.091712315877278
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,128,1,4,32767,7.080618540445964
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,128,1,8,32767,7.068501154581706
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,128,1,16,32767,7.0673065185546875
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,128,1,32,32767,7.056554794311523
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,128,1,64,32767,7.060991923014323
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,128,1,2,65535,14.156458536783854
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,128,1,1,65535,14.16977055867513
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,128,1,4,65535,14.11908213297526
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,128,1,16,65535,14.104405721028646
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,128,1,32,65535,14.101333618164062
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,128,1,8,65535,14.095359802246094
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,128,1,64,65535,14.100650787353516
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,128,1,1,65535,14.147584279378256
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,128,1,2,65535,14.147071838378906
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,128,1,8,65535,14.10321044921875
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,128,1,4,65535,14.096383412679037
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,128,1,16,65535,14.097578684488932
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,128,1,32,65535,14.076586405436197
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,128,1,64,65535,14.104063669840494
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,128,1,1,131071,28.23492177327474
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,128,1,2,131071,28.207616170247395
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,128,1,4,131071,28.191914876302082
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,128,1,8,131071,28.15112559000651
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,128,1,32,131071,28.106068929036457
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,128,1,64,131071,28.107945760091145
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,128,1,1,131071,28.246869405110676
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,128,1,16,131071,28.127914428710938
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,256,1,2,1,0.0266239990790685
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,256,1,1,1,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,256,1,4,1,0.025941332181294758
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,256,1,8,1,0.025770666698614757
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,128,1,2,131071,28.227071126302082
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,256,1,16,1,0.025258667767047882
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,256,1,32,1,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,128,1,4,131071,28.140032450358074
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,256,1,64,1,0.025941332181294758
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,256,1,1,1,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,128,1,8,131071,28.155563354492188
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,256,1,2,1,0.03379199902216593
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,256,1,4,1,0.025941332181294758
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,256,1,8,1,0.025429333249727886
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,256,1,16,1,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,256,1,32,1,0.024405332903067272
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,256,1,64,1,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,256,1,1,3,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,256,1,2,3,0.02882133424282074
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,256,1,4,3,0.025770666698614757
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,256,1,8,3,0.025429333249727886
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,256,1,16,3,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,256,1,64,3,0.025429333249727886
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,256,1,32,3,0.025258667767047882
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,256,1,1,3,0.02935466667016347
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,256,1,2,3,0.02679466704527537
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,256,1,8,3,0.02628266563018163
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,256,1,4,3,0.02628266563018163
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,128,1,32,131071,28.146347045898438
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,256,1,16,3,0.025941332181294758
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,256,1,32,3,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,128,1,64,131071,28.144126892089844
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,256,1,64,3,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,256,1,1,7,0.02935466667016347
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,256,1,2,7,0.02628266563018163
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,256,1,4,7,0.025429333249727886
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,128,1,16,131071,28.13764190673828
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,256,1,8,7,0.025429333249727886
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,256,1,16,7,0.0266239990790685
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,256,1,32,7,0.025429333249727886
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,256,1,64,7,0.025600001215934753
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,256,1,1,7,0.02918400118748347
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,256,1,2,7,0.027818667391935985
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,256,1,8,7,0.02628266563018163
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,256,1,4,7,0.025941332181294758
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,256,1,32,7,0.025429333249727886
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,256,1,16,7,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,256,1,64,7,0.025770666698614757
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,256,1,1,15,0.029696000119050343
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,256,1,4,15,0.027136000494162243
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,256,1,2,15,0.0264533335963885
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,256,1,8,15,0.025600001215934753
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,256,1,32,15,0.025770666698614757
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,256,1,64,15,0.025258667767047882
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,256,1,16,15,0.025258667767047882
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,256,1,1,15,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,256,1,2,15,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,256,1,4,15,0.02611200014750163
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,256,1,16,15,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,256,1,8,15,0.0266239990790685
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,256,1,64,15,0.025770666698614757
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,256,1,32,15,0.025429333249727886
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,256,1,2,31,0.02679466704527537
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,256,1,1,31,0.02918400118748347
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,256,1,4,31,0.027136000494162243
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,256,1,8,31,0.025429333249727886
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,256,1,16,31,0.02628266563018163
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,256,1,64,31,0.02611200014750163
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,256,1,32,31,0.0264533335963885
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,256,1,1,31,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,256,1,2,31,0.027477333943049114
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,256,1,8,31,0.0264533335963885
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,256,1,16,31,0.025941332181294758
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,256,1,32,31,0.025770666698614757
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,256,1,64,31,0.025429333249727886
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,256,1,4,31,0.02611200014750163
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,256,1,1,63,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,256,1,2,63,0.027647999425729115
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,256,1,8,63,0.027477333943049114
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,256,1,4,63,0.02679466704527537
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,256,1,16,63,0.0266239990790685
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,256,1,32,63,0.026965332527955372
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,256,1,64,63,0.02628266563018163
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,256,1,2,63,0.027818667391935985
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,256,1,4,63,0.027647999425729115
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,256,1,1,63,0.030207999050617218
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,256,1,8,63,0.027477333943049114
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,256,1,16,63,0.02628266563018163
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,256,1,32,63,0.0264533335963885
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,256,1,64,63,0.02611200014750163
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,256,1,1,127,0.05034666756788889
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,256,1,4,127,0.04727466901143392
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,256,1,2,127,0.04693333307902018
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,256,1,16,127,0.04607999821503957
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,256,1,32,127,0.04625066618124644
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,256,1,64,127,0.04625066618124644
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,256,1,1,127,0.05273599922657013
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,256,1,8,127,0.04659200211366018
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,256,1,2,127,0.048469334840774536
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,256,1,4,127,0.04744533201058706
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,256,1,8,127,0.046762665112813316
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,256,1,16,127,0.04642133414745331
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,256,1,32,127,0.04607999821503957
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,256,1,64,127,0.04625066618124644
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,256,1,1,255,0.14813866217931113
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,256,1,4,255,0.08584533135096233
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,256,1,2,255,0.08806399504343669
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,256,1,8,255,0.08721066514650981
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,256,1,16,255,0.08328533172607422
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,256,1,32,255,0.08618666728337605
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,256,1,64,255,0.08550399541854858
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,256,1,1,255,0.13431466619173685
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,256,1,2,255,0.08994133273760478
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,256,1,4,255,0.08703999718030293
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,256,1,16,255,0.08516266942024231
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,256,1,64,255,0.08584533135096233
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,256,1,32,255,0.08447999755541484
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,256,1,8,255,0.08686932921409607
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,256,1,1,511,0.2991786599159241
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,256,1,2,511,0.2667520046234131
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,256,1,4,511,0.25497599442799884
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,256,1,8,511,0.24576000372568765
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,256,1,16,511,0.24422399202982584
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,256,1,32,511,0.24115200837453207
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,256,1,64,511,0.2404693365097046
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,256,1,2,511,0.26743467648824054
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,256,1,8,511,0.24849067131678262
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,256,1,4,511,0.2529279987017314
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,256,1,1,511,0.30003199974695843
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,256,1,32,511,0.24183466037114462
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,256,1,64,511,0.24081067244211832
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,256,1,16,511,0.24422399202982584
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,256,1,1,1023,0.5265066623687744
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,256,1,2,1023,0.49664000670115155
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,256,1,4,1023,0.47940266132354736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,256,1,8,1023,0.4710400104522705
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,256,1,32,1023,0.463701327641805
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,256,1,64,1023,0.46114134788513184
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,256,1,1,1023,0.5254826545715332
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,256,1,16,1023,0.4676266511281331
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,256,1,4,1023,0.48110934098561603
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,256,1,2,1023,0.49715201059977215
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,256,1,8,1023,0.4720640182495117
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,256,1,16,1023,0.46609067916870117
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,256,1,32,1023,0.4638719956080119
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,256,1,64,1023,0.46353065967559814
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,256,1,1,2047,0.964949369430542
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,256,1,4,2047,0.9190399646759033
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,256,1,8,2047,0.9144319693247477
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,256,1,16,2047,0.9041919708251953
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,256,1,32,2047,0.9007786909739176
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,256,1,2,2047,0.9364480177561442
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,256,1,64,2047,0.8973653316497803
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,256,1,1,2047,0.9656319618225098
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,256,1,2,2047,0.9333759943644205
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,256,1,4,2047,0.920746644337972
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,256,1,8,2047,0.9105066458384196
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,256,1,32,2047,0.9004373550415039
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,256,1,16,2047,0.9053866863250732
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,256,1,1,4095,1.846442699432373
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,256,1,2,4095,1.817087968190511
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,256,1,64,2047,0.8997546831766764
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,256,1,4,4095,1.7996800740559895
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,256,1,8,4095,1.7935360272725422
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,256,1,16,4095,1.786197344462077
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,256,1,32,4095,1.780394713083903
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,256,1,64,4095,1.779882589975993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,256,1,1,4095,1.8443946838378906
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,256,1,2,4095,1.818112055460612
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,256,1,8,4095,1.7923413912455242
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,256,1,16,4095,1.7838080724080403
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,256,1,32,4095,1.7800532976786296
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,256,1,4,4095,1.80241060256958
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,256,1,64,4095,1.779882589975993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,256,1,1,8191,3.609600067138672
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,256,1,2,8191,3.5829760233561196
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,256,1,4,8191,3.568298657735189
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,256,1,8,8191,3.5558398564656577
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,256,1,16,8191,3.5459413528442383
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,256,1,32,8191,3.5406506856282554
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,256,1,1,8191,3.608917236328125
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,256,1,64,8191,3.530751864115397
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,256,1,2,8191,3.5853652954101562
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,256,1,4,8191,3.5664211908976235
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,256,1,8,8191,3.5496959686279297
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,256,1,32,8191,3.5386025110880532
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,256,1,16,8191,3.547306696573893
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,256,1,64,8191,3.539456049601237
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,256,1,1,16383,7.123285293579102
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,256,1,4,16383,7.070037206013997
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,256,1,8,16383,7.049557367960612
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,256,1,16,16383,7.047509511311849
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,256,1,2,16383,7.097855885823567
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,256,1,32,16383,7.0405120849609375
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,256,1,64,16383,7.036586761474609
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,256,1,1,16383,7.116970698038737
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,256,1,2,16383,7.076181411743164
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,256,1,4,16383,7.057578404744466
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,256,1,8,16383,7.049898783365886
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,256,1,16,16383,7.037610371907552
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,256,1,32,16383,7.026517232259114
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,256,1,64,16383,7.036757151285808
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,256,1,1,32767,14.122496287027994
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,256,1,2,32767,14.071807861328125
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,256,1,4,32767,14.052523295084635
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,256,1,8,32767,14.043818155924479
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,256,1,16,32767,14.050645192464193
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,256,1,64,32767,14.03118896484375
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,256,1,32,32767,14.028970082600912
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,256,1,2,32767,14.099285125732422
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,256,1,4,32767,14.053546905517578
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,256,1,8,32767,14.031360626220703
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,256,1,16,32767,14.040746053059896
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,256,1,32,32767,14.02077865600586
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,256,1,64,32767,14.021119435628256
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,256,1,1,32767,14.127615610758463
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,256,1,1,65535,28.11255391438802
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,256,1,2,65535,28.046847025553387
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,256,1,4,65535,28.056063334147137
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,256,1,8,65535,28.019882202148438
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,256,1,64,65535,27.975680033365887
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,256,1,16,65535,28.001108805338543
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,256,1,32,65535,28.0096435546875
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,512,1,1,1,0.054272000988324486
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,512,1,2,1,0.051541333397229515
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,256,1,2,65535,28.0446294148763
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,256,1,1,65535,28.120574951171875
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,512,1,4,1,0.04727466901143392
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,512,1,8,1,0.04710400104522705
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,512,1,16,1,0.04710400104522705
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,512,1,64,1,0.045567999283472695
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,512,1,32,1,0.04471466441949209
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,512,1,1,1,0.054101333022117615
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,256,1,4,65535,28.047190348307293
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,512,1,2,1,0.04915200173854828
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,512,1,8,1,0.04778666794300079
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,512,1,4,1,0.04625066618124644
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,512,1,16,1,0.045909335215886436
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,256,1,8,65535,28.000938415527344
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,512,1,32,1,0.04539733131726583
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,512,1,64,1,0.04607999821503957
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,256,1,16,65535,28.01220194498698
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,512,1,1,3,0.052906667192777
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,512,1,2,3,0.04915200173854828
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,512,1,4,3,0.04795733094215393
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,512,1,8,3,0.048469334840774536
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,512,1,16,3,0.04625066618124644
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,512,1,64,3,0.045226668318112694
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,512,1,32,3,0.04607999821503957
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,512,1,2,3,0.049322664737701416
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,512,1,4,3,0.05000533163547516
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,512,1,1,3,0.053930665055910744
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,256,1,32,65535,27.997355143229168
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,512,1,8,3,0.04539733131726583
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,256,1,64,65535,27.992746988932293
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,512,1,16,3,0.04539733131726583
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,512,1,64,3,0.04471466441949209
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,512,1,1,7,0.05444266895453135
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,512,1,2,7,0.049322664737701416
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,512,1,32,3,0.045226668318112694
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,512,1,4,7,0.04795733094215393
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,512,1,8,7,0.04659200211366018
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,512,1,16,7,0.04778666794300079
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,512,1,32,7,0.04744533201058706
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,512,1,2,7,0.05120000243186951
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,512,1,64,7,0.04761599997679392
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,512,1,1,7,0.05376000205675761
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,512,1,8,7,0.046762665112813316
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,512,1,4,7,0.04898133377234141
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,512,1,16,7,0.04659200211366018
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,512,1,32,7,0.045567999283472695
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,512,1,64,7,0.04625066618124644
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,512,1,1,15,0.053247998158137
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,512,1,2,15,0.05222400029500326
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,512,1,4,15,0.049322664737701416
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,512,1,8,15,0.046762665112813316
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,512,1,32,15,0.04607999821503957
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,512,1,64,15,0.045909335215886436
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,512,1,16,15,0.046762665112813316
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,512,1,1,15,0.05495466788609823
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,512,1,2,15,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,512,1,4,15,0.048298666874567665
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,512,1,8,15,0.04727466901143392
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,512,1,16,15,0.04607999821503957
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,512,1,32,15,0.04727466901143392
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,512,1,64,15,0.04744533201058706
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,512,1,4,31,0.04966400067011515
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,512,1,2,31,0.05239466826121012
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,512,1,1,31,0.05751466751098633
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,512,1,8,31,0.05017599960168203
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,512,1,16,31,0.04744533201058706
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,512,1,32,31,0.04607999821503957
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,512,1,64,31,0.048810665806134544
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,512,1,1,31,0.05853866537412008
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,512,1,2,31,0.04983466863632202
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,512,1,4,31,0.05034666756788889
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,512,1,16,31,0.04744533201058706
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,512,1,32,31,0.048810665806134544
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,512,1,64,31,0.04727466901143392
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,512,1,1,63,0.15684266885121664
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,512,1,8,31,0.04727466901143392
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,512,1,4,63,0.0506879985332489
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,512,1,2,63,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,512,1,8,63,0.05085866649945577
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,512,1,16,63,0.04983466863632202
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,512,1,32,63,0.04898133377234141
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,512,1,64,63,0.0481279989083608
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,512,1,1,63,0.15291733543078104
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,512,1,4,63,0.04983466863632202
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,512,1,2,63,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,512,1,8,63,0.04949333270390829
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,512,1,16,63,0.04966400067011515
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,512,1,32,63,0.04949333270390829
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,512,1,64,63,0.04898133377234141
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,512,1,1,127,0.23398399353027344
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,512,1,2,127,0.16315733393033346
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,512,1,4,127,0.09113599856694539
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,512,1,16,127,0.10240000486373901
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,512,1,32,127,0.0897706647713979
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,512,1,8,127,0.08840533097585042
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,512,1,64,127,0.10666666428248088
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,512,1,1,127,0.23483733336130777
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,512,1,4,127,0.09147733449935913
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,512,1,2,127,0.16520532965660095
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,512,1,8,127,0.09028266867001851
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,512,1,16,127,0.08772266904513042
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,512,1,32,127,0.08959999680519104
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,512,1,1,255,0.36369065443674725
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,512,1,2,255,0.3025919993718465
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,512,1,64,127,0.08618666728337605
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,512,1,8,255,0.2611200014750163
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,512,1,4,255,0.272213339805603
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,512,1,16,255,0.2515626748402913
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,512,1,32,255,0.2481493353843689
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,512,1,64,255,0.24388267596562704
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,512,1,1,255,0.3638613224029541
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,512,1,2,255,0.30293333530426025
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,512,1,8,255,0.25924267371495563
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,512,1,16,255,0.2515626748402913
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,512,1,4,255,0.27374933163324994
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,512,1,64,255,0.24388267596562704
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,512,1,32,255,0.2476373314857483
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,512,1,1,511,0.5894826650619507
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,512,1,2,511,0.5290666818618774
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,512,1,4,511,0.5002239942550659
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,512,1,8,511,0.4858880043029785
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,512,1,16,511,0.47547733783721924
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,512,1,64,511,0.4666026830673218
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,512,1,1,511,0.5901653369267782
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,512,1,32,511,0.47086934248606366
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,512,1,2,511,0.529749313990275
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,512,1,4,511,0.5010773340861002
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,512,1,8,511,0.48657067616780597
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,512,1,16,511,0.475818673769633
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,512,1,32,511,0.47035733858744305
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,512,1,64,511,0.4666026830673218
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,512,1,1,1023,1.0294613043467205
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,512,1,4,1023,0.9357653458913168
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,512,1,8,1023,0.9202346801757812
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,512,1,2,1023,0.9669973055521647
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,512,1,16,1023,0.9105066458384196
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,512,1,32,1023,0.9024853706359863
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,512,1,64,1023,0.899072011311849
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,512,1,1,1023,1.0316800276438396
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,512,1,2,1023,0.9668266773223877
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,512,1,4,1023,0.9381546974182129
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,512,1,8,1023,0.920746644337972
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,512,1,64,1023,0.8995839754740397
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,512,1,32,1023,0.9036800066630045
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,512,1,16,1023,0.9374720255533854
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,512,1,1,2047,1.9066880544026692
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,512,1,2,2047,1.846783955891927
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,512,1,4,2047,1.8075307210286458
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,512,1,8,2047,1.790463924407959
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,512,1,16,2047,1.779370625813802
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,512,1,32,2047,1.7722026507059734
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,512,1,64,2047,1.767082691192627
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,512,1,2,2047,1.8498560587565105
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,512,1,1,2047,1.9078826904296875
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,512,1,4,2047,1.8071893056233723
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,512,1,8,2047,1.7877333958943684
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,512,1,32,2047,1.7703253428141277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,512,1,16,2047,1.780735969543457
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,512,1,64,2047,1.7660586039225261
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,512,1,1,4095,3.6611413955688477
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,512,1,2,4095,3.5904852549235025
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,512,1,8,4095,3.535872141520182
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,512,1,16,4095,3.520341237386068
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,512,1,4,4095,3.556181271870931
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,512,1,64,4095,3.504298528035482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,512,1,32,4095,3.5111252466837564
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,512,1,1,4095,3.660458564758301
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,512,1,2,4095,3.597482681274414
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,512,1,4,4095,3.5537919998168945
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,512,1,8,4095,3.533482551574707
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,512,1,16,4095,3.51692803700765
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,512,1,64,4095,3.504810651143392
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,512,1,32,4095,3.508053461710612
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,512,1,2,8191,7.097855885823567
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,512,1,1,8191,7.179605484008789
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,512,1,4,8191,7.050069173177083
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,512,1,8,8191,7.015594482421875
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,512,1,16,8191,7.005866368611653
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,512,1,32,8191,6.985045115152995
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,512,1,64,8191,6.985898971557617
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,512,1,1,8191,7.166805267333984
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,512,1,2,8191,7.094613393147786
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,512,1,4,8191,7.047168095906575
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,512,1,8,8191,7.02395757039388
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,512,1,16,8191,7.002453486124675
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,512,1,32,8191,6.99067751566569
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,512,1,64,8191,6.994090398152669
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,512,1,1,16383,14.186837514241537
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,512,1,2,16383,14.0675417582194
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,512,1,4,16383,13.99347178141276
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,512,1,8,16383,13.976404825846354
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,512,1,16,16383,13.93442153930664
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,512,1,32,16383,13.933568318684896
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,512,1,64,16383,13.935957590738932
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,512,1,1,16383,14.146218617757162
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,512,1,2,16383,14.047402699788412
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,512,1,8,16383,13.95916748046875
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,512,1,16,16383,13.928447723388672
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,512,1,4,16383,13.980672200520834
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,512,1,32,16383,13.931690216064453
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,512,1,64,16383,13.912576039632162
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,512,1,1,32767,28.135935465494793
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,512,1,2,32767,27.970731099446613
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,512,1,4,32767,27.904340108235676
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,512,1,16,32767,27.839317321777344
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,512,1,8,32767,27.844266255696613
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,512,1,32,32767,27.794944763183594
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,512,1,64,32767,27.833513895670574
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1024,1,1,1,0.25941334168116253
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,512,1,1,32767,28.138326009114582
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1024,1,2,1,0.09403733412424724
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1024,1,4,1,0.08686932921409607
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1024,1,8,1,0.08567466338475545
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,512,1,2,32767,28.01288604736328
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1024,1,16,1,0.08311466872692108
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1024,1,32,1,0.08482133348782857
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1024,1,2,1,0.0936959981918335
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,512,1,4,32767,27.890858968098957
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,1024,1,64,1,0.0820906658967336
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1024,1,1,1,0.2563413381576538
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1024,1,4,1,0.08925867080688477
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1024,1,8,1,0.0846506655216217
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1024,1,16,1,0.08157866696516673
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1024,1,32,1,0.08226133386294048
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,512,1,8,32767,27.845120747884113
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,1024,1,64,1,0.08243200182914734
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1024,1,1,3,0.2645333409309387
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1024,1,2,3,0.09045333663622539
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1024,1,4,3,0.08772266904513042
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1024,1,8,3,0.08618666728337605
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1024,1,16,3,0.08533333738644917
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1024,1,32,3,0.08243200182914734
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,1024,1,64,3,0.08413867155710857
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1024,1,1,3,0.265557328859965
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1024,1,2,3,0.0936959981918335
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1024,1,4,3,0.08738133311271667
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1024,1,8,3,0.08516266942024231
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1024,1,16,3,0.08550399541854858
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,1024,1,64,3,0.08789333701133728
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1024,1,32,3,0.08157866696516673
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1024,1,1,7,0.274944007396698
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,512,1,32,32767,27.806719462076824
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1024,1,2,7,0.09198932846387227
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,512,1,16,32767,27.81968943277995
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1024,1,4,7,0.09233066439628601
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1024,1,8,7,0.08823466300964355
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1024,1,16,7,0.08226133386294048
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,512,1,64,32767,27.79682159423828
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1024,1,32,7,0.0846506655216217
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,1024,1,64,7,0.08260266482830048
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1024,1,2,7,0.09523199995358785
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1024,1,1,7,0.2892799973487854
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1024,1,4,7,0.091648002465566
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1024,1,8,7,0.08942932883898418
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1024,1,16,7,0.08584533135096233
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1024,1,32,7,0.08413867155710857
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,1024,1,64,7,0.08123733103275299
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1024,1,1,15,0.30856533845265705
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1024,1,4,15,0.08721066514650981
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1024,1,2,15,0.09779199957847595
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1024,1,8,15,0.09181867043177287
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1024,1,16,15,0.08721066514650981
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1024,1,32,15,0.08328533172607422
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,1024,1,64,15,0.08516266942024231
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1024,1,1,15,0.28859732548395794
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1024,1,4,15,0.09096533060073853
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1024,1,8,15,0.08806399504343669
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1024,1,2,15,0.09540266791979472
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1024,1,16,15,0.08482133348782857
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1024,1,32,15,0.08772266904513042
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,1024,1,64,15,0.08311466872692108
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1024,1,1,31,0.32290132840474445
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1024,1,2,31,0.15359999736150107
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1024,1,4,31,0.09045333663622539
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1024,1,8,31,0.09198932846387227
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1024,1,32,31,0.09028266867001851
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1024,1,16,31,0.08345599969228108
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,1024,1,64,31,0.08959999680519104
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1024,1,2,31,0.16110933820406595
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1024,1,1,31,0.31965865691502887
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1024,1,4,31,0.08823466300964355
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1024,1,8,31,0.08823466300964355
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1024,1,16,31,0.0897706647713979
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1024,1,32,31,0.08994133273760478
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1024,1,4,63,0.15923200050989786
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1024,1,1,63,0.37085866928100586
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1024,1,2,63,0.23449599742889404
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,1024,1,64,31,0.08345599969228108
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1024,1,8,63,0.09352533022562663
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1024,1,16,63,0.09062400460243225
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,1024,1,64,63,0.08994133273760478
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1024,1,32,63,0.0897706647713979
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1024,1,2,63,0.23756800095240274
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1024,1,1,63,0.3672746817270915
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1024,1,8,63,0.09079466263453166
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1024,1,32,63,0.08447999755541484
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1024,1,4,63,0.15103999773661295
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1024,1,16,63,0.09181867043177287
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,1024,1,64,63,0.09130666653315227
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1024,1,1,127,0.47701334953308105
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1024,1,2,127,0.3566933472951253
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1024,1,16,127,0.28296534220377606
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1024,1,4,127,0.29678932825724286
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1024,1,8,127,0.2701653242111206
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1024,1,32,127,0.24661332368850708
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,1024,1,64,127,0.24320000410079956
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1024,1,1,127,0.4776959816614787
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1024,1,2,127,0.3577173153559367
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1024,1,4,127,0.29713066418965656
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1024,1,8,127,0.2698240081469218
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1024,1,16,127,0.25412267446517944
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1024,1,32,127,0.24712532758712769
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,1024,1,64,127,0.24268800020217896
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1024,1,1,255,0.7156053384145101
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1024,1,2,255,0.6014293432235718
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1024,1,4,255,0.5452800194422404
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1024,1,8,255,0.5137066841125488
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1024,1,16,255,0.5207039912541708
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1024,1,32,255,0.4875946839650472
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,1024,1,64,255,0.48264535268147785
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1024,1,1,255,0.7159466743469238
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1024,1,2,255,0.603989322980245
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1024,1,4,255,0.5401599804560343
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1024,1,8,255,0.5102933247884115
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1024,1,16,255,0.4944213231404622
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1024,1,32,255,0.48520533243815106
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,1024,1,64,255,0.4828159809112549
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1024,1,1,511,1.1805013020833333
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1024,1,2,511,1.0605226357777913
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1024,1,4,511,1.0011306603749592
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1024,1,8,511,0.9646080334981283
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1024,1,16,511,0.942250649134318
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1024,1,32,511,0.9384960333506266
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,1024,1,64,511,0.9296212991078695
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1024,1,1,511,1.1827200253804524
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1024,1,4,511,0.9943040211995443
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1024,1,2,511,1.0533546606699626
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1024,1,8,511,0.9661440054575602
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1024,1,16,511,0.942250649134318
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,1024,1,64,511,0.932522694269816
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1024,1,32,511,0.9321813583374023
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1024,1,1,1023,2.0980052947998047
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1024,1,2,1023,1.9485012690226238
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1024,1,4,1023,1.898837407430013
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1024,1,8,1023,1.8604373931884766
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1024,1,16,1023,1.8379093805948894
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1024,1,32,1023,1.8186240196228027
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,1024,1,64,1023,1.8133333524068196
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1024,1,1,1023,2.102442741394043
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1024,1,4,1023,1.9031039873758953
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1024,1,2,1023,1.969663937886556
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1024,1,8,1023,1.8546346028645833
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1024,1,16,1023,1.8428586324055989
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1024,1,32,1023,1.8232320149739583
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,1024,1,64,1023,1.8148694038391113
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1024,1,1,2047,3.867306709289551
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1024,1,2,2047,3.7608105341593423
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1024,1,4,2047,3.6817919413248696
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1024,1,8,2047,3.641002655029297
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1024,1,16,2047,3.610111872355143
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1024,1,32,2047,3.5978240966796875
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,1024,1,64,2047,3.595605214436849
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1024,1,1,2047,3.877034823099772
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1024,1,2,2047,3.7500588099161782
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1024,1,4,2047,3.6817919413248696
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1024,1,8,2047,3.647146542867025
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1024,1,16,2047,3.6164267857869468
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1024,1,32,2047,3.595434824625651
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,1024,1,64,2047,3.589632034301758
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1024,1,1,4095,7.49124272664388
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1024,1,2,4095,7.2910505930582685
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1024,1,4,4095,7.247701644897461
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1024,1,8,4095,7.177728017171224
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1024,1,16,4095,7.164927800496419
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1024,1,32,4095,7.156394958496094
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,1024,1,64,4095,7.136938730875651
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1024,1,1,4095,7.428607940673828
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1024,1,2,4095,7.314090728759766
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1024,1,4,4095,7.208618799845378
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1024,1,8,4095,7.2005971272786455
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1024,1,16,4095,7.178581237792969
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1024,1,32,4095,7.143594741821289
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,1024,1,64,4095,7.1470082600911455
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1024,1,1,8191,14.583807627360025
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1024,1,2,8191,14.400341033935547
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1024,1,8,8191,14.201002756754557
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1024,1,4,8191,14.326613108317057
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1024,1,16,8191,14.238719940185547
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1024,1,32,8191,14.220458984375
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,1024,1,64,8191,14.211072285970053
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1024,1,1,8191,14.54916254679362
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1024,1,2,8191,14.388565063476562
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1024,1,4,8191,14.267903645833334
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1024,1,8,8191,14.25595728556315
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1024,1,16,8191,14.211413065592447
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1024,1,32,8191,14.224554697672525
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,1024,1,64,8191,14.23138173421224
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1024,1,1,16383,28.715349833170574
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1024,1,2,16383,28.52556864420573
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1024,1,4,16383,28.384597778320312
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1024,1,8,16383,28.375722249348957
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1024,1,32,16383,28.340736389160156
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1024,1,16,16383,28.374526977539062
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,1,1024,1,64,16383,28.286463419596355
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1024,1,1,16383,28.658004760742188
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1,1,2,1,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,1,1,1,1,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1,1,8,1,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1,1,4,1,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1,1,16,1,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1,1,32,1,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1,1,64,1,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,1,1,1,1,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1,1,2,1,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1,1,4,1,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1,1,8,1,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1,1,16,1,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1,1,32,1,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1,1,64,1,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,1,1,1,3,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1,1,2,3,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1,1,4,3,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1,1,8,3,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1,1,16,3,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1,1,32,3,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1,1,64,3,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,1,1,1,3,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1024,1,2,16383,28.53136952718099
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1,1,2,3,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1,1,4,3,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1,1,8,3,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1,1,16,3,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1024,1,4,16383,28.40661366780599
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1,1,32,3,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1,1,64,3,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,1,1,1,7,0.012837332983811697
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1,1,4,7,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1,1,2,7,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1,1,8,7,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1,1,16,7,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1,1,32,7,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1,1,64,7,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,1,1,1,7,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1,1,2,7,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1,1,4,7,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1,1,8,7,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1,1,16,7,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1024,1,8,16383,28.373675028483074
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1,1,32,7,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1,1,64,7,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,1,1,1,15,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1,1,2,15,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1,1,4,15,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1,1,8,15,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1,1,64,15,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1,1,32,15,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1,1,16,15,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,1,1,1,15,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1,1,2,15,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1,1,4,15,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1,1,8,15,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1,1,16,15,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1024,1,16,16383,28.35968017578125
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1,1,64,15,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1,1,32,15,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,1,1,1,31,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1,1,2,31,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1,1,4,31,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1,1,8,31,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1,1,16,31,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,1,1024,1,64,16383,28.293291727701824
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1024,1,32,16383,28.309674580891926
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1,1,32,31,0.01022933361430963
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,1,1,1,31,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1,1,64,31,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1,1,2,31,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1,1,4,31,0.010543999572594961
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1,1,8,31,0.01073066641887029
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1,1,16,31,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1,1,32,31,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1,1,4,63,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,1,1,1,63,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1,1,64,31,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1,1,8,63,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1,1,16,63,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1,1,2,63,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1,1,32,63,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1,1,64,63,0.012250666817029318
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,1,1,1,63,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1,1,2,63,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1,1,8,63,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1,1,4,63,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1,1,16,63,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1,1,64,63,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1,1,32,63,0.0103946669648091
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,1,1,1,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1,1,4,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1,1,2,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1,1,8,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1,1,16,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1,1,64,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1,1,32,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,1,1,1,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1,1,2,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1,1,8,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1,1,4,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1,1,32,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1,1,16,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1,1,64,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,1,1,1,255,0.024746666351954143
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1,1,2,255,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1,1,4,255,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1,1,8,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1,1,16,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1,1,64,255,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1,1,32,255,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,1,1,1,255,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1,1,4,255,0.024405332903067272
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1,1,2,255,0.024405332903067272
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1,1,8,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1,1,16,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1,1,32,255,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1,1,64,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1,1,2,511,0.04437333345413208
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,1,1,1,511,0.04454400142033895
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1,1,4,511,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1,1,8,511,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1,1,16,511,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1,1,32,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1,1,64,511,0.04369066655635834
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,1,1,1,511,0.04454400142033895
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1,1,2,511,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1,1,4,511,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1,1,16,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1,1,8,511,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1,1,32,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1,1,64,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,1,1,1,1023,0.08413867155710857
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1,1,2,1023,0.08396800359090169
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1,1,4,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1,1,8,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1,1,16,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1,1,32,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1,1,64,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,1,1,1,1023,0.08447999755541484
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1,1,4,1023,0.08396800359090169
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1,1,2,1023,0.08396800359090169
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1,1,8,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1,1,16,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1,1,32,1023,0.08345599969228108
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1,1,64,1023,0.08345599969228108
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,1,1,1,2047,0.16383999586105347
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1,1,8,2047,0.1634986698627472
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1,1,2,2047,0.16366933782895407
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1,1,4,2047,0.16332800189654031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1,1,16,2047,0.16332800189654031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1,1,32,2047,0.16315733393033346
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1,1,64,2047,0.1629866659641266
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,1,1,1,2047,0.16383999586105347
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1,1,4,2047,0.16332800189654031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1,1,2,2047,0.16366933782895407
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1,1,8,2047,0.16332800189654031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1,1,16,2047,0.16315733393033346
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1,1,32,2047,0.16315733393033346
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1,1,64,2047,0.1629866659641266
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,1,1,1,4095,0.3227306604385376
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1,1,2,4095,0.32255999247233075
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1,1,4,4095,0.32204800844192505
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1,1,8,4095,0.32238932450612384
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1,1,64,4095,0.32204800844192505
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1,1,16,4095,0.3222186764081319
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1,1,32,4095,0.3222186764081319
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1,1,4,4095,0.32238932450612384
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,1,1,1,4095,0.3227306604385376
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1,1,8,4095,0.3227306604385376
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1,1,2,4095,0.32255999247233075
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1,1,16,4095,0.32238932450612384
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1,1,64,4095,0.32204800844192505
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1,1,32,4095,0.32204800844192505
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,1,1,1,8191,0.6435840129852295
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1,1,4,8191,0.640341321627299
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1,1,2,8191,0.6406826575597128
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1,1,16,8191,0.6398293177286783
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1,1,8,8191,0.6398293177286783
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1,1,32,8191,0.640341321627299
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,1,1,1,8191,0.6406826575597128
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1,1,2,8191,0.6405119895935059
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1,1,64,8191,0.6401706536610922
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1,1,8,8191,0.640341321627299
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1,1,4,8191,0.6401706536610922
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1,1,16,8191,0.6398293177286783
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1,1,32,8191,0.6396586497624716
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1,1,64,8191,0.640341321627299
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,1,1,1,16383,1.2767573197682698
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1,1,2,16383,1.276245355606079
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1,1,4,16383,1.2760746479034424
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1,1,8,16383,1.276245355606079
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1,1,16,16383,1.2757333119710286
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1,1,64,16383,1.2757333119710286
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1,1,32,16383,1.2755626837412517
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,1,1,1,16383,1.276586691538493
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1,1,2,16383,1.2764159838358562
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1,1,4,16383,1.2760746479034424
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1,1,8,16383,1.2759040196736653
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1,1,16,16383,1.2760746479034424
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1,1,32,16383,1.2755626837412517
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1,1,64,16383,1.2755626837412517
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,1,1,1,32767,2.549077351888021
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1,1,2,32767,2.54805326461792
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1,1,4,32767,2.54805326461792
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1,1,8,32767,2.5473705927530923
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1,1,16,32767,2.5473705927530923
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1,1,32,32767,2.547541300455729
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1,1,64,32767,2.5470293362935386
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,1,1,1,32767,2.5487359364827475
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1,1,2,32767,2.5482239723205566
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1,1,4,32767,2.5478827158610025
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1,1,8,32767,2.5478827158610025
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1,1,16,32767,2.5472000439961753
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1,1,32,32767,2.5472000439961753
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1,1,64,32767,2.547541300455729
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,1,1,1,65535,5.0920108159383135
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1,1,2,65535,5.092522621154785
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1,1,4,65535,5.09166940053304
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1,1,8,65535,5.090986569722493
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1,1,16,65535,5.090986569722493
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1,1,32,65535,5.089621225992839
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1,1,64,65535,5.089621225992839
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,1,1,1,65535,5.0920108159383135
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1,1,4,65535,5.091498692830403
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1,1,2,65535,5.089962641398112
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1,1,8,65535,5.091498692830403
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1,1,16,65535,5.090986569722493
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1,1,32,65535,5.0908158620198565
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1,1,64,65535,5.089450518290202
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,1,1,1,131071,10.368170420328775
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1,1,2,131071,10.393941243489584
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1,1,4,131071,10.364757537841797
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1,1,16,131071,10.355029424031576
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1,1,8,131071,10.356053034464518
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1,1,32,131071,10.36031977335612
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,1,1,1,131071,10.358442942301432
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1,1,64,131071,10.358783721923828
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1,1,2,131071,10.375850677490234
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,2,1,1,1,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1,1,4,131071,10.378922780354818
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1,1,8,131071,10.359125137329102
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,2,1,2,1,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1,1,16,131071,10.356053034464518
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1,1,32,131071,10.35434659322103
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1,1,64,131071,10.361514409383139
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,2,1,4,1,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,2,1,8,1,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,2,1,16,1,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,2,1,32,1,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,2,1,1,1,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,2,1,64,1,0.008874666566650072
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,2,1,4,1,0.010527999450763067
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,2,1,2,1,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,2,1,16,1,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,2,1,8,1,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,2,1,32,1,0.010378666842977205
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,2,1,64,1,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,2,1,1,3,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,2,1,2,3,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,2,1,4,3,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,2,1,8,3,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,2,1,16,3,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,2,1,64,3,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,2,1,32,3,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,2,1,1,3,0.010911999891201654
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,2,1,2,3,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,2,1,4,3,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,2,1,8,3,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,2,1,16,3,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,2,1,32,3,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,2,1,64,3,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,2,1,2,7,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,2,1,4,7,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,2,1,1,7,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,2,1,8,7,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,2,1,32,7,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,2,1,16,7,0.010714666297038397
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,2,1,64,7,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,2,1,1,7,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,2,1,2,7,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,2,1,4,7,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,2,1,8,7,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,2,1,16,7,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,2,1,32,7,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,2,1,64,7,0.012655999511480331
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,2,1,1,15,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,2,1,4,15,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,2,1,2,15,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,2,1,8,15,0.010543999572594961
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,2,1,16,15,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,2,1,32,15,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,2,1,64,15,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,2,1,1,15,0.011050666371981302
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,2,1,2,15,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,2,1,4,15,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,2,1,8,15,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,2,1,16,15,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,2,1,32,15,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,2,1,64,15,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,2,1,1,31,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,2,1,2,31,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,2,1,4,31,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,2,1,8,31,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,2,1,32,31,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,2,1,16,31,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,2,1,64,31,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,2,1,1,31,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,2,1,2,31,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,2,1,4,31,0.010533332824707031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,2,1,8,31,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,2,1,16,31,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,2,1,64,31,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,2,1,32,31,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,2,1,1,63,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,2,1,2,63,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,2,1,4,63,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,2,1,8,63,0.011077333241701126
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,2,1,16,63,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,2,1,32,63,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,2,1,64,63,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,2,1,1,63,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,2,1,2,63,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,2,1,4,63,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,2,1,8,63,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,2,1,16,63,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,2,1,32,63,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,2,1,64,63,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,2,1,1,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,2,1,2,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,2,1,4,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,2,1,8,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,2,1,16,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,2,1,32,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,2,1,64,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,2,1,1,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,2,1,2,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,2,1,4,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,2,1,8,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,2,1,16,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,2,1,32,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,2,1,64,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,2,1,1,255,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,2,1,2,255,0.024405332903067272
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,2,1,4,255,0.024405332903067272
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,2,1,8,255,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,2,1,16,255,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,2,1,32,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,2,1,64,255,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,2,1,1,255,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,2,1,2,255,0.024405332903067272
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,2,1,4,255,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,2,1,8,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,2,1,16,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,2,1,32,255,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,2,1,64,255,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,2,1,1,511,0.04454400142033895
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,2,1,2,511,0.04437333345413208
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,2,1,4,511,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,2,1,8,511,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,2,1,16,511,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,2,1,32,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,2,1,64,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,2,1,1,511,0.04437333345413208
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,2,1,2,511,0.04437333345413208
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,2,1,8,511,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,2,1,4,511,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,2,1,16,511,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,2,1,32,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,2,1,64,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,2,1,2,1023,0.08413867155710857
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,2,1,1,1023,0.08430932958920796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,2,1,4,1023,0.08396800359090169
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,2,1,8,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,2,1,16,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,2,1,32,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,2,1,64,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,2,1,1,1023,0.08430932958920796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,2,1,2,1023,0.08413867155710857
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,2,1,4,1023,0.08396800359090169
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,2,1,8,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,2,1,16,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,2,1,32,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,2,1,64,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,2,1,2,2047,0.16366933782895407
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,2,1,1,2047,0.16366933782895407
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,2,1,4,2047,0.16332800189654031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,2,1,16,2047,0.16315733393033346
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,2,1,8,2047,0.16332800189654031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,2,1,32,2047,0.1629866659641266
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,2,1,64,2047,0.16315733393033346
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,2,1,1,2047,0.16383999586105347
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,2,1,2,2047,0.1634986698627472
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,2,1,4,2047,0.16332800189654031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,2,1,8,2047,0.16332800189654031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,2,1,16,2047,0.16332800189654031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,2,1,32,2047,0.1629866659641266
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,2,1,64,2047,0.16315733393033346
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,2,1,1,4095,0.32290132840474445
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,2,1,2,4095,0.32255999247233075
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,2,1,4,4095,0.32238932450612384
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,2,1,8,4095,0.32238932450612384
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,2,1,16,4095,0.3222186764081319
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,2,1,32,4095,0.32204800844192505
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,2,1,1,4095,0.32290132840474445
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,2,1,64,4095,0.32204800844192505
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,2,1,4,4095,0.32238932450612384
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,2,1,2,4095,0.32255999247233075
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,2,1,8,4095,0.32238932450612384
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,2,1,16,4095,0.3222186764081319
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,2,1,32,4095,0.32204800844192505
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,2,1,1,8191,0.6410239934921265
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,2,1,64,4095,0.32204800844192505
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,2,1,2,8191,0.6405119895935059
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,2,1,4,8191,0.640341321627299
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,2,1,8,8191,0.6405119895935059
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,2,1,16,8191,0.6405119895935059
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,2,1,32,8191,0.640341321627299
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,2,1,64,8191,0.6401706536610922
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,2,1,1,8191,0.6408533255259196
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,2,1,4,8191,0.640341321627299
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,2,1,2,8191,0.6405119895935059
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,2,1,8,8191,0.6401706536610922
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,2,1,64,8191,0.640341321627299
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,2,1,16,8191,0.6405119895935059
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,2,1,1,16383,1.276586691538493
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,2,1,2,16383,1.2764159838358562
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,2,1,4,16383,1.2764159838358562
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,2,1,8,16383,1.2760746479034424
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,2,1,16,16383,1.2760746479034424
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,2,1,32,16383,1.2759040196736653
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,2,1,32,8191,0.6401706536610922
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,2,1,64,16383,1.2760746479034424
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,2,1,1,16383,1.2767573197682698
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,2,1,2,16383,1.276586691538493
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,2,1,4,16383,1.276245355606079
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,2,1,8,16383,1.2760746479034424
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,2,1,16,16383,1.2760746479034424
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,2,1,32,16383,1.2760746479034424
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,2,1,64,16383,1.276245355606079
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,2,1,1,32767,2.5483946800231934
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,2,1,2,32767,2.5482239723205566
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,2,1,4,32767,2.54805326461792
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,2,1,8,32767,2.54805326461792
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,2,1,16,32767,2.5478827158610025
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,2,1,32,32767,2.5478827158610025
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,2,1,64,32767,2.547541300455729
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,2,1,1,32767,2.549077351888021
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,2,1,2,32767,2.5482239723205566
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,2,1,4,32767,2.54805326461792
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,2,1,8,32767,2.5478827158610025
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,2,1,16,32767,2.5478827158610025
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,2,1,64,32767,2.5477120081583657
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,2,1,32,32767,2.5478827158610025
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,2,1,1,65535,5.194410641988118
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,2,1,2,65535,5.20908800760905
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,2,1,4,65535,5.188949267069499
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,2,1,8,65535,5.185536066691081
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,2,1,32,65535,5.18178145090739
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,2,1,16,65535,5.180586814880371
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,2,1,64,65535,5.187072118123372
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,2,1,1,65535,5.191679954528809
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,2,1,2,65535,5.200384140014648
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,2,1,4,65535,5.194069226582845
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,2,1,8,65535,5.1843414306640625
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,2,1,16,65535,5.18178145090739
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,2,1,64,65535,5.179562568664551
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,2,1,32,65535,5.182122548421224
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,2,1,1,131071,10.381653467814127
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,2,1,2,131071,10.401962916056315
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,2,1,4,131071,10.382848103841146
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,2,1,8,131071,10.369194666544596
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,2,1,32,131071,10.359978357950846
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,2,1,16,131071,10.365269343058268
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,2,1,64,131071,10.35690689086914
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,2,1,1,131071,10.37397321065267
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,2,1,2,131071,10.40725326538086
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,2,1,4,131071,10.388992309570312
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,2,1,8,131071,10.373119990030924
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,4,1,1,1,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,2,1,16,131071,10.365610758463541
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,4,1,2,1,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,2,1,32,131071,10.361002604166666
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,2,1,64,131071,10.364245096842447
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,4,1,4,1,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,4,1,8,1,0.01055466632048289
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,4,1,16,1,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,4,1,32,1,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,4,1,64,1,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,4,1,1,1,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,4,1,2,1,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,4,1,4,1,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,4,1,8,1,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,4,1,16,1,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,4,1,32,1,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,4,1,64,1,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,4,1,4,3,0.010559999694426855
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,4,1,1,3,0.010890666395425797
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,4,1,2,3,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,4,1,8,3,0.011258666714032492
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,4,1,16,3,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,4,1,32,3,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,4,1,1,3,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,4,1,64,3,0.010186666622757912
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,4,1,2,3,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,4,1,4,3,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,4,1,8,3,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,4,1,16,3,0.010693332801262537
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,4,1,32,3,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,4,1,64,3,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,4,1,1,7,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,4,1,2,7,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,4,1,4,7,0.01055466632048289
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,4,1,8,7,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,4,1,16,7,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,4,1,32,7,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,4,1,2,7,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,4,1,4,7,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,4,1,1,7,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,4,1,8,7,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,4,1,64,7,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,4,1,16,7,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,4,1,64,7,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,4,1,32,7,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,4,1,1,15,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,4,1,2,15,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,4,1,8,15,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,4,1,4,15,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,4,1,16,15,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,4,1,64,15,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,4,1,32,15,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,4,1,1,15,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,4,1,2,15,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,4,1,4,15,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,4,1,8,15,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,4,1,16,15,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,4,1,32,15,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,4,1,64,15,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,4,1,1,31,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,4,1,2,31,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,4,1,4,31,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,4,1,8,31,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,4,1,16,31,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,4,1,64,31,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,4,1,32,31,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,4,1,1,31,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,4,1,2,31,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,4,1,4,31,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,4,1,8,31,0.010559999694426855
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,4,1,16,31,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,4,1,32,31,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,4,1,64,31,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,4,1,2,63,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,4,1,1,63,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,4,1,4,63,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,4,1,8,63,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,4,1,16,63,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,4,1,32,63,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,4,1,64,63,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,4,1,1,63,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,4,1,2,63,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,4,1,8,63,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,4,1,16,63,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,4,1,4,63,0.012495999534924826
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,4,1,32,63,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,4,1,64,63,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,4,1,1,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,4,1,2,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,4,1,4,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,4,1,8,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,4,1,16,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,4,1,1,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,4,1,64,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,4,1,32,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,4,1,2,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,4,1,4,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,4,1,8,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,4,1,32,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,4,1,16,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,4,1,64,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,4,1,1,255,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,4,1,2,255,0.024405332903067272
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,4,1,4,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,4,1,8,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,4,1,32,255,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,4,1,16,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,4,1,64,255,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,4,1,1,255,0.024746666351954143
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,4,1,2,255,0.024405332903067272
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,4,1,8,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,4,1,4,255,0.024405332903067272
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,4,1,16,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,4,1,32,255,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,4,1,64,255,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,4,1,1,511,0.04437333345413208
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,4,1,4,511,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,4,1,2,511,0.04437333345413208
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,4,1,8,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,4,1,16,511,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,4,1,32,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,4,1,64,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,4,1,2,511,0.04437333345413208
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,4,1,1,511,0.04437333345413208
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,4,1,4,511,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,4,1,8,511,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,4,1,16,511,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,4,1,64,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,4,1,32,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,4,1,1,1023,0.08430932958920796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,4,1,2,1023,0.08396800359090169
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,4,1,8,1023,0.08396800359090169
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,4,1,4,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,4,1,16,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,4,1,32,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,4,1,64,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,4,1,1,1023,0.08430932958920796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,4,1,2,1023,0.08413867155710857
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,4,1,4,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,4,1,8,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,4,1,16,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,4,1,32,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,4,1,64,1023,0.08345599969228108
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,4,1,1,2047,0.16366933782895407
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,4,1,2,2047,0.1634986698627472
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,4,1,8,2047,0.16332800189654031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,4,1,4,2047,0.1634986698627472
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,4,1,16,2047,0.16332800189654031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,4,1,32,2047,0.16315733393033346
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,4,1,64,2047,0.16315733393033346
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,4,1,1,2047,0.16383999586105347
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,4,1,2,2047,0.16366933782895407
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,4,1,32,2047,0.1629866659641266
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,4,1,4,2047,0.16332800189654031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,4,1,8,2047,0.16332800189654031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,4,1,16,2047,0.16332800189654031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,4,1,64,2047,0.16315733393033346
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,4,1,1,4095,0.32290132840474445
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,4,1,4,4095,0.32255999247233075
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,4,1,2,4095,0.32255999247233075
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,4,1,8,4095,0.32238932450612384
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,4,1,16,4095,0.32238932450612384
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,4,1,32,4095,0.32204800844192505
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,4,1,64,4095,0.3222186764081319
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,4,1,1,4095,0.3227306604385376
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,4,1,2,4095,0.3227306604385376
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,4,1,4,4095,0.32238932450612384
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,4,1,8,4095,0.3222186764081319
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,4,1,16,4095,0.3222186764081319
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,4,1,32,4095,0.32204800844192505
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,4,1,1,8191,0.6408533255259196
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,4,1,2,8191,0.6406826575597128
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,4,1,4,8191,0.6405119895935059
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,4,1,16,8191,0.640341321627299
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,4,1,8,8191,0.640341321627299
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,4,1,64,4095,0.32204800844192505
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,4,1,32,8191,0.6399999856948853
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,4,1,64,8191,0.6399999856948853
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,4,1,1,8191,0.6410239934921265
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,4,1,4,8191,0.6405119895935059
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,4,1,8,8191,0.640341321627299
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,4,1,2,8191,0.6408533255259196
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,4,1,32,8191,0.6401706536610922
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,4,1,64,8191,0.6399999856948853
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,4,1,16,8191,0.640341321627299
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,4,1,1,16383,1.2767573197682698
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,4,1,2,16383,1.276586691538493
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,4,1,4,16383,1.2764159838358562
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,4,1,16,16383,1.276245355606079
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,4,1,32,16383,1.2760746479034424
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,4,1,4,16383,1.276245355606079
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,4,1,8,16383,1.276245355606079
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,4,1,64,16383,1.2759040196736653
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,4,1,1,16383,1.2774399916330974
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,4,1,2,16383,1.2764159838358562
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,4,1,8,16383,1.2760746479034424
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,4,1,32,16383,1.2760746479034424
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,4,1,16,16383,1.2760746479034424
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,4,1,64,16383,1.2759040196736653
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,4,1,2,32767,2.6002772649129233
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,4,1,1,32767,2.6023252805074057
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,4,1,4,32767,2.5992533365885415
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,4,1,8,32767,2.5975467363993325
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,4,1,16,32767,2.59822940826416
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,4,1,32,32767,2.6170026461283364
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,4,1,64,32767,2.59549872080485
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,4,1,2,32767,2.6007893880208335
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,4,1,1,32767,2.6006186803181968
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,4,1,4,32767,2.6002772649129233
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,4,1,8,32767,2.5978879928588867
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,4,1,16,32767,2.5961813926696777
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,4,1,64,32767,2.5958399772644043
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,4,1,32,32767,2.5978879928588867
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,4,1,2,65535,5.199872016906738
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,4,1,1,65535,5.20362663269043
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,4,1,4,65535,5.193045298258464
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,4,1,8,65535,5.1877546310424805
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,4,1,16,65535,5.190314610799153
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,4,1,32,65535,5.18724250793457
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,4,1,64,65535,5.185365358988444
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,4,1,1,65535,5.200554529825847
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,4,1,2,65535,5.198335965474446
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,4,1,4,65535,5.196288108825684
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,4,1,8,65535,5.186559995015462
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,4,1,16,65535,5.1858774820963545
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,4,1,32,65535,5.188096046447754
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,4,1,64,65535,5.186730702718099
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,4,1,1,131071,10.388309478759766
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,4,1,2,131071,10.406229019165039
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,4,1,4,131071,10.372949600219727
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,4,1,8,131071,10.374314626057943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,4,1,16,131071,10.366463979085287
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,4,1,32,131071,10.365952173868815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,4,1,64,131071,10.37124252319336
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,4,1,1,131071,10.386773427327475
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,8,1,1,1,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,4,1,2,131071,10.40281613667806
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,4,1,4,131071,10.394282658894857
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,4,1,8,131071,10.365439732869467
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,4,1,16,131071,10.373290379842123
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,4,1,32,131071,10.36509895324707
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,4,1,64,131071,10.36680539449056
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,8,1,2,1,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,8,1,4,1,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,8,1,8,1,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,8,1,16,1,0.010586666564146677
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,8,1,32,1,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,8,1,1,1,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,8,1,64,1,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,8,1,2,1,0.010863999525705973
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,8,1,4,1,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,8,1,8,1,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,8,1,16,1,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,8,1,32,1,0.010357333347201347
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,8,1,64,1,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,8,1,1,3,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,8,1,2,3,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,8,1,4,3,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,8,1,8,3,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,8,1,16,3,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,8,1,32,3,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,8,1,64,3,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,8,1,2,3,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,8,1,1,3,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,8,1,4,3,0.01071999967098236
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,8,1,8,3,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,8,1,32,3,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,8,1,16,3,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,8,1,1,7,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,8,1,64,3,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,8,1,2,7,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,8,1,4,7,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,8,1,8,7,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,8,1,16,7,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,8,1,32,7,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,8,1,64,7,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,8,1,2,7,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,8,1,1,7,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,8,1,4,7,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,8,1,16,7,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,8,1,8,7,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,8,1,32,7,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,8,1,64,7,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,8,1,1,15,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,8,1,4,15,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,8,1,2,15,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,8,1,8,15,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,8,1,16,15,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,8,1,32,15,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,8,1,64,15,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,8,1,1,15,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,8,1,2,15,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,8,1,4,15,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,8,1,8,15,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,8,1,16,15,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,8,1,32,15,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,8,1,64,15,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,8,1,1,31,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,8,1,2,31,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,8,1,8,31,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,8,1,4,31,0.010890666395425797
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,8,1,16,31,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,8,1,32,31,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,8,1,64,31,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,8,1,1,31,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,8,1,2,31,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,8,1,4,31,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,8,1,8,31,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,8,1,16,31,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,8,1,32,31,0.01003200002014637
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,8,1,64,31,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,8,1,1,63,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,8,1,2,63,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,8,1,4,63,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,8,1,8,63,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,8,1,16,63,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,8,1,64,63,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,8,1,32,63,0.010197333370645842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,8,1,1,63,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,8,1,4,63,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,8,1,2,63,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,8,1,16,63,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,8,1,8,63,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,8,1,32,63,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,8,1,64,63,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,8,1,1,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,8,1,2,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,8,1,4,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,8,1,8,127,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,8,1,16,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,8,1,32,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,8,1,64,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,8,1,1,127,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,8,1,2,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,8,1,8,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,8,1,4,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,8,1,16,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,8,1,32,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,8,1,64,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,8,1,1,255,0.024746666351954143
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,8,1,2,255,0.024405332903067272
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,8,1,8,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,8,1,4,255,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,8,1,32,255,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,8,1,16,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,8,1,64,255,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,8,1,1,255,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,8,1,2,255,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,8,1,4,255,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,8,1,8,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,8,1,32,255,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,8,1,16,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,8,1,64,255,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,8,1,1,511,0.04471466441949209
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,8,1,2,511,0.04437333345413208
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,8,1,4,511,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,8,1,8,511,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,8,1,16,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,8,1,32,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,8,1,64,511,0.04369066655635834
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,8,1,1,511,0.04454400142033895
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,8,1,2,511,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,8,1,4,511,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,8,1,8,511,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,8,1,16,511,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,8,1,32,511,0.04369066655635834
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,8,1,64,511,0.04369066655635834
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,8,1,1,1023,0.08430932958920796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,8,1,2,1023,0.08413867155710857
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,8,1,4,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,8,1,8,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,8,1,16,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,8,1,32,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,8,1,64,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,8,1,1,1023,0.08430932958920796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,8,1,2,1023,0.08413867155710857
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,8,1,4,1023,0.08396800359090169
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,8,1,8,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,8,1,16,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,8,1,64,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,8,1,32,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,8,1,1,2047,0.16401066382726034
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,8,1,2,2047,0.1634986698627472
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,8,1,8,2047,0.1634986698627472
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,8,1,16,2047,0.16315733393033346
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,8,1,4,2047,0.1634986698627472
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,8,1,32,2047,0.1629866659641266
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,8,1,64,2047,0.1629866659641266
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,8,1,1,2047,0.16401066382726034
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,8,1,2,2047,0.16366933782895407
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,8,1,4,2047,0.16332800189654031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,8,1,8,2047,0.16332800189654031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,8,1,16,2047,0.1634986698627472
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,8,1,64,2047,0.1629866659641266
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,8,1,32,2047,0.1629866659641266
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,8,1,1,4095,0.32290132840474445
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,8,1,2,4095,0.32255999247233075
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,8,1,4,4095,0.32238932450612384
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,8,1,8,4095,0.32238932450612384
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,8,1,16,4095,0.3222186764081319
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,8,1,32,4095,0.3222186764081319
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,8,1,64,4095,0.32204800844192505
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,8,1,2,4095,0.32255999247233075
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,8,1,1,4095,0.32290132840474445
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,8,1,4,4095,0.32238932450612384
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,8,1,8,4095,0.3222186764081319
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,8,1,16,4095,0.32238932450612384
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,8,1,64,4095,0.3222186764081319
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,8,1,32,4095,0.3222186764081319
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,8,1,1,8191,0.6406826575597128
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,8,1,2,8191,0.6405119895935059
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,8,1,4,8191,0.6714026927947998
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,8,1,8,8191,0.640341321627299
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,8,1,16,8191,0.6408533255259196
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,8,1,32,8191,0.640341321627299
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,8,1,64,8191,0.6401706536610922
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,8,1,1,8191,0.6408533255259196
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,8,1,2,8191,0.6405119895935059
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,8,1,4,8191,0.6574079990386963
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,8,1,8,8191,0.6401706536610922
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,8,1,16,8191,0.6401706536610922
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,8,1,64,8191,0.640341321627299
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,8,1,32,8191,0.6399999856948853
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,8,1,1,16383,1.3083306948343914
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,8,1,2,16383,1.3079893589019775
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,8,1,4,16383,1.3047466278076172
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,8,1,8,16383,1.3032106558481853
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,8,1,16,16383,1.3023573557535808
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,8,1,32,16383,1.3023573557535808
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,8,1,2,16383,1.3078186511993408
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,8,1,64,16383,1.3025279839833577
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,8,1,1,16383,1.3071359793345134
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,8,1,4,16383,1.305087963740031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,8,1,8,16383,1.3026986916859944
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,8,1,16,16383,1.3025279839833577
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,8,1,32,16383,1.3023573557535808
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,8,1,64,16383,1.302186648050944
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,8,1,1,32767,2.6077866554260254
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,8,1,4,32767,2.6009599367777505
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,8,1,8,32767,2.59822940826416
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,8,1,2,32767,2.612053394317627
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,8,1,16,32767,2.6006186803181968
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,8,1,32,32767,2.597376028696696
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,8,1,64,32767,2.599082628885905
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,8,1,1,32767,2.612394650777181
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,8,1,2,32767,2.6088107426961265
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,8,1,4,32767,2.60044797261556
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,8,1,8,32767,2.598741372426351
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,8,1,32,32767,2.5985706647237143
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,8,1,64,32767,2.5968640645345054
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,8,1,16,32767,2.599936008453369
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,8,1,2,65535,5.214890797932942
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,8,1,1,65535,5.210794766743978
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,8,1,4,65535,5.20140806833903
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,8,1,8,65535,5.1923627853393555
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,8,1,16,65535,5.192192077636719
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,8,1,32,65535,5.192874590555827
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,8,1,64,65535,5.19048531850179
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,8,1,1,65535,5.2191572189331055
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,8,1,2,65535,5.2157440185546875
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,8,1,8,65535,5.194410641988118
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,8,1,4,65535,5.2031145095825195
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,8,1,16,65535,5.18997319539388
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,8,1,32,65535,5.189632097880046
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,8,1,64,65535,5.192192077636719
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,8,1,1,131071,10.401450475056967
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,8,1,2,131071,10.427733103434244
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,8,1,4,131071,10.404693603515625
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,8,1,8,131071,10.38097063700358
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,8,1,32,131071,10.371754964192709
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,8,1,16,131071,10.379093170166016
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,8,1,64,131071,10.37277857462565
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,8,1,1,131071,10.414592107137045
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,8,1,2,131071,10.409642537434896
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,16,1,1,1,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,8,1,4,131071,10.401280085245768
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,16,1,2,1,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,8,1,8,131071,10.390527725219727
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,8,1,16,131071,10.376192092895508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,8,1,32,131071,10.372437159220377
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,8,1,64,131071,10.378751754760742
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,16,1,4,1,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,16,1,8,1,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,16,1,16,1,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,16,1,32,1,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,16,1,64,1,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,16,1,1,1,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,16,1,2,1,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,16,1,8,1,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,16,1,4,1,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,16,1,16,1,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,16,1,64,1,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,16,1,32,1,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,16,1,1,3,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,16,1,4,3,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,16,1,2,3,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,16,1,8,3,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,16,1,16,3,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,16,1,32,3,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,16,1,64,3,0.011098666737476984
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,16,1,1,3,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,16,1,2,3,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,16,1,4,3,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,16,1,8,3,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,16,1,16,3,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,16,1,32,3,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,16,1,1,7,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,16,1,64,3,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,16,1,2,7,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,16,1,4,7,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,16,1,8,7,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,16,1,16,7,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,16,1,32,7,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,16,1,1,7,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,16,1,2,7,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,16,1,4,7,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,16,1,64,7,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,16,1,8,7,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,16,1,16,7,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,16,1,64,7,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,16,1,32,7,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,16,1,1,15,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,16,1,2,15,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,16,1,4,15,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,16,1,16,15,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,16,1,32,15,0.010202666744589806
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,16,1,1,15,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,16,1,64,15,0.009045333291093508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,16,1,2,15,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,16,1,4,15,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,16,1,8,15,0.0102186668664217
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,16,1,8,15,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,16,1,32,15,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,16,1,16,15,0.011765333513418833
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,16,1,64,15,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,16,1,1,31,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,16,1,2,31,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,16,1,4,31,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,16,1,16,31,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,16,1,32,31,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,16,1,64,31,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,16,1,1,31,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,16,1,8,31,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,16,1,2,31,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,16,1,4,31,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,16,1,8,31,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,16,1,16,31,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,16,1,32,31,0.010538666198650995
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,16,1,64,31,0.010559999694426855
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,16,1,1,63,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,16,1,8,63,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,16,1,2,63,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,16,1,4,63,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,16,1,16,63,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,16,1,1,63,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,16,1,64,63,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,16,1,2,63,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,16,1,32,63,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,16,1,4,63,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,16,1,64,63,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,16,1,16,63,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,16,1,32,63,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,16,1,8,63,0.010741333166758219
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,16,1,1,127,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,16,1,2,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,16,1,4,127,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,16,1,8,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,16,1,16,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,16,1,64,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,16,1,32,127,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,16,1,1,127,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,16,1,2,127,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,16,1,4,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,16,1,8,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,16,1,16,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,16,1,32,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,16,1,64,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,16,1,1,255,0.025258667767047882
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,16,1,4,255,0.024405332903067272
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,16,1,8,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,16,1,2,255,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,16,1,16,255,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,16,1,64,255,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,16,1,32,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,16,1,1,255,0.025258667767047882
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,16,1,2,255,0.024405332903067272
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,16,1,4,255,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,16,1,32,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,16,1,16,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,16,1,8,255,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,16,1,64,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,16,1,2,511,0.04454400142033895
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,16,1,1,511,0.04505600035190582
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,16,1,4,511,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,16,1,8,511,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,16,1,16,511,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,16,1,1,511,0.04505600035190582
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,16,1,32,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,16,1,64,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,16,1,2,511,0.04454400142033895
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,16,1,4,511,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,16,1,16,511,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,16,1,8,511,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,16,1,32,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,16,1,64,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,16,1,2,1023,0.08430932958920796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,16,1,1,1023,0.08499200145403545
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,16,1,4,1023,0.08396800359090169
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,16,1,8,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,16,1,16,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,16,1,32,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,16,1,64,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,16,1,1,1023,0.08482133348782857
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,16,1,2,1023,0.08430932958920796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,16,1,8,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,16,1,16,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,16,1,4,1023,0.08396800359090169
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,16,1,32,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,16,1,64,1023,0.08362666765848796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,16,1,1,2047,0.16452266772588095
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,16,1,2,2047,0.16366933782895407
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,16,1,4,2047,0.1634986698627472
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,16,1,8,2047,0.16332800189654031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,16,1,64,2047,0.1629866659641266
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,16,1,16,2047,0.16315733393033346
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,16,1,32,2047,0.1629866659641266
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,16,1,1,2047,0.16452266772588095
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,16,1,2,2047,0.16366933782895407
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,16,1,4,2047,0.1634986698627472
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,16,1,8,2047,0.1634986698627472
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,16,1,16,2047,0.16332800189654031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,16,1,32,2047,0.1629866659641266
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,16,1,64,2047,0.1629866659641266
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,16,1,2,4095,0.3227306604385376
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,16,1,1,4095,0.32392533620198566
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,16,1,4,4095,0.32238932450612384
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,16,1,8,4095,0.32238932450612384
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,16,1,16,4095,0.32255999247233075
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,16,1,32,4095,0.32204800844192505
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,16,1,64,4095,0.32204800844192505
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,16,1,1,4095,0.32358400026957196
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,16,1,2,4095,0.3227306604385376
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,16,1,8,4095,0.32238932450612384
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,16,1,4,4095,0.32238932450612384
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,16,1,16,4095,0.32238932450612384
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,16,1,32,4095,0.32238932450612384
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,16,1,64,4095,0.32204800844192505
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,16,1,1,8191,0.6666239897410074
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,16,1,4,8191,0.6601386864980062
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,16,1,2,8191,0.6661119858423868
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,16,1,8,8191,0.6599680185317993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,16,1,16,8191,0.6586026748021444
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,16,1,64,8191,0.6574079990386963
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,16,1,32,8191,0.658240000406901
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,16,1,1,8191,0.6650880177815756
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,16,1,2,8191,0.6654293139775594
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,16,1,4,8191,0.6599680185317993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,16,1,8,8191,0.6601386864980062
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,16,1,16,8191,0.6592853466669718
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,16,1,64,8191,0.6575786670049032
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,16,1,32,8191,0.6592853466669718
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,16,1,1,16383,1.3214720090230305
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,16,1,2,16383,1.3195947011311848
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,16,1,4,16383,1.3146453698476155
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,16,1,8,16383,1.310208002726237
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,16,1,16,16383,1.3115733464558919
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,16,1,32,16383,1.3115733464558919
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,16,1,64,16383,1.3083306948343914
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,16,1,1,16383,1.3214720090230305
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,16,1,2,16383,1.3189120292663574
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,16,1,4,16383,1.3136213620503743
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,16,1,8,16383,1.3120853106180828
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,16,1,16,16383,1.3091839949289958
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,16,1,32,16383,1.3119146823883057
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,16,1,64,16383,1.3112320105234783
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,16,1,1,32767,2.627925395965576
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,16,1,2,32767,2.632362683614095
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,16,1,8,32767,2.6139307022094727
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,16,1,4,32767,2.64516274134318
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,16,1,16,32767,2.6163199742635093
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,16,1,32,32767,2.610858599344889
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,16,1,64,32767,2.618879954020182
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,16,1,2,32767,2.624512036641439
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,16,1,1,32767,2.6383360226949057
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,16,1,8,32767,2.613589286804199
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,16,1,4,32767,2.621781349182129
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,16,1,32,32767,2.6152960459391275
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,16,1,16,32767,2.6127360661824546
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,16,1,64,32767,2.6105173428853354
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,16,1,1,65535,5.260287920633952
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,16,1,4,65535,5.233493487040202
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,16,1,2,65535,5.235712051391602
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,16,1,8,65535,5.21830399831136
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,16,1,32,65535,5.220693270365397
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,16,1,16,65535,5.218133290608724
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,16,1,64,65535,5.216767946879069
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,16,1,1,65535,5.238954544067383
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,16,1,2,65535,5.256874720255534
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,16,1,8,65535,5.2123308181762695
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,16,1,4,65535,5.232981363932292
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,16,1,16,65535,5.215231895446777
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,16,1,32,65535,5.220181465148926
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,16,1,64,65535,5.216597239176433
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,16,1,1,131071,10.492928187052408
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,16,1,2,131071,10.464256286621094
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,16,1,4,131071,10.447872161865234
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,16,1,16,131071,10.432682673136393
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,16,1,8,131071,10.417322794596354
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,16,1,32,131071,10.423125584920248
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,16,1,64,131071,10.434730529785156
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,16,1,1,131071,10.479957580566406
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,16,1,2,131071,10.483882904052734
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,16,1,4,131071,10.43609619140625
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,32,1,1,1,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,16,1,8,131071,10.43780263264974
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,32,1,2,1,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,16,1,16,131071,10.428927739461264
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,16,1,32,131071,10.41100819905599
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,16,1,64,131071,10.42141850789388
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,32,1,4,1,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,32,1,8,1,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,32,1,16,1,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,32,1,64,1,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,32,1,32,1,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,32,1,1,1,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,32,1,2,1,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,32,1,4,1,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,32,1,8,1,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,32,1,16,1,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,32,1,32,1,0.010373333469033241
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,32,1,1,3,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,32,1,64,1,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,32,1,4,3,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,32,1,2,3,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,32,1,8,3,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,32,1,16,3,0.01055466632048289
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,32,1,32,3,0.01073066641887029
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,32,1,64,3,0.010869332899649939
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,32,1,1,3,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,32,1,2,3,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,32,1,4,3,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,32,1,8,3,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,32,1,16,3,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,32,1,32,3,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,32,1,1,7,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,32,1,64,3,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,32,1,2,7,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,32,1,4,7,0.010890666395425797
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,32,1,8,7,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,32,1,16,7,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,32,1,32,7,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,32,1,64,7,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,32,1,1,7,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,32,1,2,7,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,32,1,4,7,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,32,1,8,7,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,32,1,16,7,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,32,1,32,7,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,32,1,64,7,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,32,1,1,15,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,32,1,2,15,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,32,1,4,15,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,32,1,8,15,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,32,1,32,15,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,32,1,16,15,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,32,1,64,15,0.01126933346192042
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,32,1,1,15,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,32,1,2,15,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,32,1,4,15,0.01108266661564509
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,32,1,8,15,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,32,1,16,15,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,32,1,32,15,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,32,1,1,31,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,32,1,64,15,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,32,1,2,31,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,32,1,4,31,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,32,1,8,31,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,32,1,16,31,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,32,1,32,31,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,32,1,64,31,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,32,1,1,31,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,32,1,2,31,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,32,1,4,31,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,32,1,8,31,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,32,1,16,31,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,32,1,32,31,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,32,1,64,31,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,32,1,2,63,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,32,1,1,63,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,32,1,8,63,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,32,1,4,63,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,32,1,16,63,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,32,1,32,63,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,32,1,1,63,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,32,1,64,63,0.009216000015536943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,32,1,2,63,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,32,1,4,63,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,32,1,8,63,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,32,1,16,63,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,32,1,32,63,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,32,1,64,63,0.00938666673998038
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,32,1,1,127,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,32,1,2,127,0.0170666662355264
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,32,1,4,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,32,1,16,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,32,1,8,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,32,1,1,127,0.022357332209746044
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,32,1,32,127,0.02628266563018163
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,32,1,2,127,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,32,1,64,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,32,1,4,127,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,32,1,8,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,32,1,16,127,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,32,1,32,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,32,1,64,127,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,32,1,2,255,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,32,1,1,255,0.03822933385769526
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,32,1,4,255,0.024405332903067272
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,32,1,8,255,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,32,1,16,255,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,32,1,32,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,32,1,64,255,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,32,1,1,255,0.03857066730658213
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,32,1,2,255,0.17567465702692667
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,32,1,4,255,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,32,1,8,255,0.024405332903067272
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,32,1,16,255,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,32,1,32,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,32,1,64,255,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,32,1,1,511,0.07031466563542683
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,32,1,2,511,0.04505600035190582
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,32,1,4,511,0.04437333345413208
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,32,1,16,511,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,32,1,8,511,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,32,1,32,511,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,32,1,64,511,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,32,1,1,511,0.07014399766921997
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,32,1,2,511,0.04505600035190582
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,32,1,4,511,0.04437333345413208
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,32,1,8,511,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,32,1,16,511,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,32,1,64,511,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,32,1,32,511,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,32,1,1,1023,0.13516799608866373
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,32,1,2,1023,0.08482133348782857
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,32,1,4,1023,0.08413867155710857
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,32,1,8,1023,0.08413867155710857
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,32,1,16,1023,0.08396800359090169
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,32,1,32,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,32,1,64,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,32,1,2,1023,0.0846506655216217
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,32,1,1,1023,0.1360213359196981
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,32,1,4,1023,0.08430932958920796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,32,1,8,1023,0.08413867155710857
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,32,1,16,1023,0.08413867155710857
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,32,1,32,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,32,1,64,1023,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,32,1,2,2047,0.1641813317934672
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,32,1,1,2047,0.2611200014750163
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,32,1,4,2047,0.16383999586105347
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,32,1,8,2047,0.1634986698627472
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,32,1,16,2047,0.16366933782895407
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,32,1,32,2047,0.16332800189654031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,32,1,64,2047,0.16332800189654031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,32,1,1,2047,0.26180267333984375
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,32,1,2,2047,0.16435199975967407
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,32,1,4,2047,0.16366933782895407
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,32,1,16,2047,0.1634986698627472
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,32,1,8,2047,0.1634986698627472
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,32,1,32,2047,0.1634986698627472
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,32,1,64,2047,0.16332800189654031
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,32,1,1,4095,0.5169493357340494
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,32,1,2,4095,0.3546453317006429
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,32,1,4,4095,0.3536213239034017
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,32,1,8,4095,0.34986666838328045
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,32,1,16,4095,0.34935466448465985
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,32,1,32,4095,0.35140268007914227
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,32,1,1,4095,0.5184853474299113
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,32,1,64,4095,0.3495253324508667
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,32,1,2,4095,0.3573760191599528
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,32,1,4,4095,0.35037867228190106
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,32,1,8,4095,0.35072000821431476
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,32,1,16,4095,0.3495253324508667
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,32,1,32,4095,0.3485013246536255
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,32,1,64,4095,0.34935466448465985
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,32,1,2,8191,0.6971733570098877
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,32,1,1,8191,1.0236586729685466
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,32,1,4,8191,0.6932480335235596
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,32,1,16,8191,0.6913706461588541
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,32,1,8,8191,0.688810666402181
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,32,1,32,8191,0.6900053024291992
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,32,1,64,8191,0.6850559711456299
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,32,1,2,8191,0.697002649307251
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,32,1,4,8191,0.6937599976857504
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,32,1,1,8191,1.021440029144287
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,32,1,8,8191,0.6905173460642496
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,32,1,16,8191,0.6910293102264404
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,32,1,64,8191,0.688810666402181
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,32,1,32,8191,0.688810666402181
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,32,1,1,16383,2.0377599398295083
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,32,1,2,16383,1.3783040046691895
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,32,1,4,16383,1.3666987419128418
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,32,1,16,16383,1.3646507263183594
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,32,1,8,16383,1.3617493311564128
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,32,1,32,16383,1.36516269048055
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,32,1,64,16383,1.3586773872375488
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,32,1,1,16383,2.0392959912618003
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,32,1,2,16383,1.3714772860209148
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,32,1,8,16383,1.3619200388590496
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,32,1,4,16383,1.3658453623453777
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,32,1,16,16383,1.3634559313456218
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,32,1,32,16383,1.3607254028320312
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,32,1,64,16383,1.3641386032104492
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,32,1,1,32767,4.064597447713216
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,32,1,2,32767,2.717866579691569
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,32,1,4,32767,2.7055788040161133
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,32,1,8,32767,2.7060906092325845
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,32,1,16,32767,2.700458526611328
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,32,1,32,32767,2.716671943664551
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,32,1,64,32767,2.7093334197998047
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,32,1,1,32767,4.084394772847493
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,32,1,2,32767,2.731861432393392
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,32,1,4,32767,2.6975574493408203
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,32,1,8,32767,2.6973867416381836
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,32,1,32,32767,2.7019945780436196
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,32,1,16,32767,2.709162712097168
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,32,1,64,32767,2.7136001586914062
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,32,1,1,65535,8.130730946858725
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,32,1,2,65535,5.455872217814128
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,32,1,4,65535,5.41542371114095
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,32,1,8,65535,5.381120045979817
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,32,1,16,65535,5.367808024088542
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,32,1,64,65535,5.401941299438477
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,32,1,32,65535,5.3896535237630205
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,32,1,1,65535,8.145407994588217
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,32,1,2,65535,5.465258916219075
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,32,1,4,65535,5.421567916870117
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,32,1,8,65535,5.397162755330403
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,32,1,16,65535,5.371562957763672
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,32,1,32,65535,5.364053090413411
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,32,1,64,65535,5.400917053222656
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,32,1,2,131071,10.909183502197266
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,32,1,1,131071,16.24285888671875
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,32,1,4,131071,10.809343973795572
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,32,1,8,131071,10.787498474121094
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,32,1,16,131071,10.770773569742838
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,32,1,32,131071,10.729642232259115
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,32,1,64,131071,10.715136210123697
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,32,1,1,131071,16.28945032755534
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,64,1,1,1,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,32,1,2,131071,10.909695943196615
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,32,1,4,131071,10.779135386149088
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,32,1,8,131071,10.777088165283203
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,32,1,16,131071,10.78988774617513
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,64,1,2,1,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,32,1,64,131071,10.725716908772787
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,32,1,32,131071,10.754901885986328
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,64,1,4,1,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,64,1,8,1,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,64,1,16,1,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,64,1,64,1,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,64,1,1,1,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,64,1,32,1,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,64,1,2,1,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,64,1,4,1,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,64,1,8,1,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,64,1,16,1,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,64,1,32,1,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,64,1,64,1,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,64,1,1,3,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,64,1,2,3,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,64,1,4,3,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,64,1,8,3,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,64,1,32,3,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,64,1,16,3,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,64,1,64,3,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,64,1,1,3,0.016895999511082966
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,64,1,2,3,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,64,1,4,3,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,64,1,8,3,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,64,1,16,3,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,64,1,32,3,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,64,1,64,3,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,64,1,1,7,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,64,1,2,7,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,64,1,4,7,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,64,1,8,7,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,64,1,16,7,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,64,1,32,7,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,64,1,64,7,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,64,1,1,7,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,64,1,2,7,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,64,1,4,7,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,64,1,8,7,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,64,1,16,7,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,64,1,32,7,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,64,1,1,15,0.016895999511082966
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,64,1,64,7,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,64,1,2,15,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,64,1,4,15,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,64,1,8,15,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,64,1,16,15,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,64,1,32,15,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,64,1,64,15,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,64,1,2,15,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,64,1,1,15,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,64,1,4,15,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,64,1,8,15,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,64,1,16,15,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,64,1,32,15,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,64,1,64,15,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,64,1,2,31,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,64,1,4,31,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,64,1,1,31,0.016895999511082966
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,64,1,16,31,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,64,1,8,31,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,64,1,32,31,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,64,1,64,31,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,64,1,4,31,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,64,1,1,31,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,64,1,2,31,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,64,1,8,31,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,64,1,16,31,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,64,1,64,31,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,64,1,32,31,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,64,1,1,63,0.017407999684413273
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,64,1,2,63,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,64,1,4,63,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,64,1,32,63,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,64,1,8,63,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,64,1,16,63,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,64,1,1,63,0.017237332959969837
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,64,1,64,63,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,64,1,2,63,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,64,1,4,63,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,64,1,8,63,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,64,1,16,63,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,64,1,2,127,0.022015998760859173
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,64,1,64,63,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,64,1,4,127,0.021503999829292297
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,64,1,8,127,0.021162666380405426
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,64,1,32,63,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,64,1,1,127,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,64,1,32,127,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,64,1,16,127,0.021162666380405426
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,64,1,1,127,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,64,1,64,127,0.020992000897725422
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,64,1,2,127,0.022015998760859173
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,64,1,16,127,0.020992000897725422
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,64,1,32,127,0.020992000897725422
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,64,1,8,127,0.021162666380405426
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,64,1,2,255,0.03788800040880839
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,64,1,1,255,0.04966400067011515
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,64,1,64,127,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,64,1,4,127,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,64,1,4,255,0.03754666695992152
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,64,1,8,255,0.037205333511034645
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,64,1,32,255,0.03669333209594091
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,64,1,16,255,0.036864000062147774
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,64,1,64,255,0.036864000062147774
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,64,1,1,255,0.04949333270390829
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,64,1,2,255,0.03788800040880839
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,64,1,8,255,0.03737599899371465
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,64,1,16,255,0.03703466554482778
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,64,1,32,255,0.036864000062147774
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,64,1,64,255,0.036864000062147774
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,64,1,1,511,0.09301333626111348
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,64,1,2,511,0.06980266670385997
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,64,1,4,255,0.03754666695992152
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,64,1,4,511,0.06963199873765309
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,64,1,8,511,0.06911999980608623
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,64,1,16,511,0.06877866884072621
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,64,1,32,511,0.06860800087451935
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,64,1,64,511,0.06843733290831248
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,64,1,1,511,0.09403733412424724
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,64,1,2,511,0.06963199873765309
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,64,1,8,511,0.06911999980608623
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,64,1,16,511,0.06877866884072621
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,64,1,32,511,0.0682666649421056
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,64,1,64,511,0.06843733290831248
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,64,1,2,1023,0.13499733805656433
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,64,1,1,1023,0.17407999436060587
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,64,1,4,511,0.06963199873765309
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,64,1,4,1023,0.13346133629480997
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,64,1,8,1023,0.13260799646377563
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,64,1,16,1023,0.13192533453305563
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,64,1,32,1023,0.13141333063443503
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,64,1,64,1023,0.13141333063443503
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,64,1,1,1023,0.18039466937383017
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,64,1,2,1023,0.13516799608866373
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,64,1,8,1023,0.13243732849756876
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,64,1,4,1023,0.13363200426101685
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,64,1,16,1023,0.13209600249926248
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,64,1,32,1023,0.13175466656684875
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,64,1,64,1023,0.13141333063443503
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,64,1,1,2047,0.3474773168563843
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,64,1,2,2047,0.2657279968261719
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,64,1,4,2047,0.26333866516749066
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,64,1,8,2047,0.26231465737024945
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,64,1,32,2047,0.26333866516749066
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,64,1,1,2047,0.3479893207550049
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,64,1,64,2047,0.26282666126887005
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,64,1,4,2047,0.26316799720128375
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,64,1,2,2047,0.26658133665720624
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,64,1,8,2047,0.26333866516749066
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,64,1,16,2047,0.2629973292350769
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,64,1,16,2047,0.2629973292350769
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,64,1,32,2047,0.26282666126887005
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,64,1,2,4095,0.5188266833623251
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,64,1,1,4095,0.687445322672526
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,64,1,64,2047,0.26282666126887005
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,64,1,4,4095,0.514901320139567
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,64,1,8,4095,0.5152426560719808
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,64,1,16,4095,0.5145599842071533
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,64,1,64,4095,0.5147306521733602
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,64,1,1,4095,0.688981294631958
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,64,1,2,4095,0.5189973513285319
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,64,1,4,4095,0.5147306521733602
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,64,1,8,4095,0.5142186482747396
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,64,1,16,4095,0.5145599842071533
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,64,1,32,4095,0.5142186482747396
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,64,1,32,4095,0.5135360161463419
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,64,1,64,4095,0.5140479803085327
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,64,1,16,8191,1.0337279637654622
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,64,1,1,8191,1.4035627047220867
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,64,1,4,8191,1.031167984008789
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,64,1,2,8191,1.0390186309814453
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,64,1,8,8191,1.0327040354410808
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,64,1,64,8191,1.0327040354410808
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,64,1,1,8191,1.3928106625874836
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,64,1,32,8191,1.0308266480763753
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,64,1,2,8191,1.0426026980082195
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,64,1,4,8191,1.0427733262379963
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,64,1,8,8191,1.0303146839141846
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,64,1,16,8191,1.0327040354410808
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,64,1,32,8191,1.0337279637654622
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,64,1,64,8191,1.0338986714680989
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,64,1,1,16383,2.7397120793660483
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,64,1,8,16383,2.1695146560668945
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,64,1,4,16383,2.1712212562561035
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,64,1,16,16383,2.1659305890401206
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,64,1,32,16383,2.167466640472412
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,64,1,1,16383,2.821802775065104
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,64,1,64,16383,2.164906660715739
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,64,1,2,16383,2.17958402633667
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,64,1,2,16383,2.189141273498535
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,64,1,4,16383,2.171903928120931
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,64,1,8,16383,2.167125384012858
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,64,1,16,16383,2.165760040283203
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,64,1,32,16383,2.1655893325805664
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,64,1,64,16383,2.1705387433369956
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,64,1,1,32767,5.160277366638184
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,64,1,2,32767,4.509525299072266
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,64,1,4,32767,4.500480016072591
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,64,1,8,32767,4.488362630208333
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,64,1,16,32767,4.478805224100749
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,64,1,32,32767,4.474879900614421
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,64,1,64,32767,4.4772694905598955
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,64,1,1,32767,5.291861216227214
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,64,1,2,32767,4.517546653747559
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,64,1,4,32767,4.485461235046387
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,64,1,16,32767,4.490240097045898
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,64,1,32,32767,4.466346740722656
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,64,1,64,32767,4.468736012776692
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,64,1,8,32767,4.487850824991862
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,64,1,1,65535,10.734762827555338
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,64,1,2,65535,9.201663970947266
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,64,1,4,65535,9.161898930867514
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,64,1,8,65535,9.151317596435547
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,64,1,16,65535,9.148074467976889
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,64,1,64,65535,9.122133255004883
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,64,1,32,65535,9.137493133544922
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,64,1,1,65535,10.65233039855957
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,64,1,2,65535,9.202005386352539
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,64,1,4,65535,9.155413309733072
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,64,1,8,65535,9.135445276896158
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,64,1,16,65535,9.12452252705892
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,64,1,32,65535,9.1419308980306
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,64,1,64,65535,9.123157501220703
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,64,1,1,131071,21.469014485677082
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,64,1,4,131071,18.396160125732422
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,64,1,16,131071,18.342912038167317
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,64,1,8,131071,18.45794169108073
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,64,1,2,131071,18.51528549194336
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,64,1,32,131071,18.444800059000652
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,64,1,64,131071,18.425003051757812
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,64,1,1,131071,21.340159098307293
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,128,1,1,1,0.028501334289709728
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,128,1,2,1,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,64,1,2,131071,18.552661895751953
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,128,1,4,1,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,128,1,8,1,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,64,1,4,131071,18.40674082438151
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,64,1,8,131071,18.458282470703125
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,64,1,16,131071,18.368512471516926
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,128,1,16,1,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,128,1,32,1,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,64,1,64,131071,18.394794464111328
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,64,1,32,131071,18.45077387491862
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,128,1,64,1,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,128,1,4,1,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,128,1,2,1,0.016895999511082966
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,128,1,1,1,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,128,1,16,1,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,128,1,8,1,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,128,1,32,1,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,128,1,64,1,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,128,1,2,3,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,128,1,4,3,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,128,1,8,3,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,128,1,1,3,0.02918400118748347
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,128,1,16,3,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,128,1,32,3,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,128,1,64,3,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,128,1,2,3,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,128,1,1,3,0.029525332152843475
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,128,1,4,3,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,128,1,8,3,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,128,1,64,3,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,128,1,32,3,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,128,1,1,7,0.02918400118748347
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,128,1,16,3,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,128,1,2,7,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,128,1,8,7,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,128,1,4,7,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,128,1,16,7,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,128,1,32,7,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,128,1,1,7,0.029525332152843475
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,128,1,64,7,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,128,1,4,7,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,128,1,8,7,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,128,1,2,7,0.016895999511082966
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,128,1,32,7,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,128,1,16,7,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,128,1,1,15,0.028501334289709728
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,128,1,64,7,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,128,1,2,15,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,128,1,4,15,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,128,1,8,15,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,128,1,32,15,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,128,1,16,15,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,128,1,64,15,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,128,1,1,15,0.029525332152843475
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,128,1,4,15,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,128,1,2,15,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,128,1,8,15,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,128,1,16,15,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,128,1,32,15,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,128,1,2,31,0.016895999511082966
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,128,1,1,31,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,128,1,8,31,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,128,1,64,15,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,128,1,4,31,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,128,1,16,31,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,128,1,32,31,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,128,1,64,31,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,128,1,1,31,0.02918400118748347
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,128,1,2,31,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,128,1,4,31,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,128,1,16,31,0.016384000579516094
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,128,1,32,31,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,128,1,64,31,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,128,1,1,63,0.031061333914597828
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,128,1,8,31,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,128,1,2,63,0.016895999511082966
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,128,1,4,63,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,128,1,8,63,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,128,1,16,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,128,1,32,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,128,1,64,63,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,128,1,1,63,0.030720000465710957
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,128,1,4,63,0.016042667130629223
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,128,1,2,63,0.0170666662355264
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,128,1,16,63,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,128,1,8,63,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,128,1,32,63,0.015018666783968607
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,128,1,64,63,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,128,1,1,127,0.05120000243186951
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,128,1,2,127,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,128,1,4,127,0.025941332181294758
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,128,1,16,127,0.024746666351954143
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,128,1,8,127,0.02611200014750163
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,128,1,32,127,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,128,1,64,127,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,128,1,1,127,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,128,1,8,127,0.025258667767047882
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,128,1,4,127,0.02611200014750163
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,128,1,2,127,0.028501334289709728
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,128,1,16,127,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,128,1,1,255,0.09352533022562663
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,128,1,64,127,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,128,1,32,127,0.024746666351954143
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,128,1,2,255,0.048469334840774536
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,128,1,4,255,0.04625066618124644
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,128,1,8,255,0.04454400142033895
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,128,1,16,255,0.04488533238569895
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,128,1,32,255,0.04454400142033895
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,128,1,64,255,0.04386133452256521
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,128,1,1,255,0.09198932846387227
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,128,1,4,255,0.04505600035190582
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,128,1,8,255,0.04505600035190582
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,128,1,16,255,0.04505600035190582
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,128,1,32,255,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,128,1,2,255,0.049322664737701416
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,128,1,64,255,0.04454400142033895
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,128,1,1,511,0.18551466862360635
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,128,1,4,511,0.08618666728337605
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,128,1,8,511,0.10410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,128,1,2,511,0.09318400422732036
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,128,1,16,511,0.08533333738644917
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,128,1,32,511,0.08345599969228108
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,128,1,1,511,0.1868799924850464
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,128,1,64,511,0.08482133348782857
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,128,1,2,511,0.0942080020904541
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,128,1,4,511,0.08447999755541484
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,128,1,8,511,0.08533333738644917
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,128,1,16,511,0.08584533135096233
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,128,1,32,511,0.08345599969228108
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,128,1,64,511,0.08345599969228108
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,128,1,1,1023,0.37546666463216144
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,128,1,4,1023,0.2501973311106364
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,128,1,8,1023,0.24661332368850708
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,128,1,16,1023,0.24439465999603271
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,128,1,2,1023,0.25804799795150757
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,128,1,32,1023,0.24234666426976523
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,128,1,64,1023,0.24217599630355835
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,128,1,1,1023,0.3752959966659546
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,128,1,2,1023,0.26026666164398193
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,128,1,4,1023,0.250709335009257
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,128,1,8,1023,0.2474666635195414
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,128,1,16,1023,0.24371200799942017
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,128,1,64,1023,0.24149332443873087
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,128,1,1,2047,0.7057066758473715
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,128,1,32,1023,0.24234666426976523
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,128,1,2,2047,0.4802560011545817
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,128,1,4,2047,0.4734293222427368
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,128,1,8,2047,0.46865065892537433
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,128,1,16,2047,0.4652373393376668
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,128,1,32,2047,0.46455466747283936
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,128,1,64,2047,0.4623359839121501
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,128,1,1,2047,0.7024640242258707
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,128,1,4,2047,0.48349865277608234
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,128,1,8,2047,0.4676266511281331
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,128,1,2,2047,0.48162134488423664
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,128,1,16,2047,0.46557867527008057
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,128,1,64,2047,0.4626773198445638
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,128,1,32,2047,0.4643839995066325
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,128,1,1,4095,1.379327932993571
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,128,1,2,4095,0.9238186677296957
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,128,1,4,4095,0.9157973130544027
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,128,1,32,4095,0.9036800066630045
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,128,1,8,4095,0.9079466660817465
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,128,1,64,4095,0.9007786909739176
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,128,1,16,4095,0.904533306757609
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,128,1,1,4095,1.374037265777588
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,128,1,2,4095,0.9250133037567139
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,128,1,8,4095,0.908799966176351
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,128,1,4,4095,0.9169920285542806
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,128,1,16,4095,0.907093365987142
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,128,1,32,4095,0.9023146629333496
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,128,1,64,4095,0.9014613628387451
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,128,1,2,8191,1.8082133928934734
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,128,1,1,8191,2.6695680618286133
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,128,1,4,8191,1.8015573819478352
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,128,1,8,8191,1.7930240631103516
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,128,1,16,8191,1.7897814114888508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,128,1,32,8191,1.7914880116780598
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,128,1,1,8191,2.7729921340942383
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,128,1,64,8191,1.7841493288675945
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,128,1,2,8191,1.806165377298991
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,128,1,8,8191,1.7957545916239421
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,128,1,16,8191,1.790463924407959
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,128,1,32,8191,1.7890987396240234
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,128,1,4,8191,1.7979733149210613
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,128,1,64,8191,1.7890987396240234
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,128,1,1,16383,5.254144032796224
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,128,1,2,16383,3.576832135518392
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,128,1,4,16383,3.5703465143839517
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,128,1,8,16383,3.566762606302897
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,128,1,16,16383,3.551402727762858
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,128,1,32,16383,3.554645220438639
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,128,1,1,16383,5.168981234232585
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,128,1,64,16383,3.55020809173584
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,128,1,2,16383,3.5845120747884116
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,128,1,4,16383,3.5691518783569336
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,128,1,8,16383,3.5638612111409507
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,128,1,16,16383,3.562154769897461
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,128,1,32,16383,3.5554987589518228
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,128,1,64,16383,3.5493545532226562
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,128,1,1,32767,10.422101338704428
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,128,1,2,32767,7.1178239186604815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,128,1,4,32767,7.089834849039714
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,128,1,8,32767,7.082154591878255
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,128,1,16,32767,7.070037206013997
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,128,1,64,32767,7.0698668162028
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,128,1,32,32767,7.065088272094727
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,128,1,1,32767,10.541397094726562
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,128,1,2,32767,7.110314687093099
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,128,1,8,32767,7.069695790608724
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,128,1,16,32767,7.067989349365234
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,128,1,4,32767,7.082325617472331
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,128,1,32,32767,7.06065050760905
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,128,1,64,32767,7.06935437520345
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,128,1,2,65535,14.161749521891275
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,128,1,1,65535,21.069482167561848
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,128,1,8,65535,14.120277404785156
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,128,1,4,65535,14.131712595621744
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,128,1,16,65535,14.107306162516275
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,128,1,64,65535,14.090240478515625
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,128,1,32,65535,14.102869669596354
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,128,1,1,65535,21.003946940104168
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,128,1,2,65535,14.158335367838541
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,128,1,4,65535,14.139391581217447
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,128,1,8,65535,14.120618184407553
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,128,1,16,65535,14.090410868326822
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,128,1,64,65535,14.071125030517578
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,128,1,32,65535,14.0948486328125
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,128,1,2,131071,28.238848368326824
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,128,1,1,131071,42.38284810384115
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,128,1,8,131071,28.162729899088543
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,128,1,4,131071,28.234580993652344
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,128,1,16,131071,28.17535909016927
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,128,1,32,131071,28.12774403889974
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,128,1,64,131071,28.119552612304688
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,128,1,1,131071,42.36799875895182
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,256,1,1,1,0.054272000988324486
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,256,1,2,1,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,256,1,4,1,0.0264533335963885
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,256,1,8,1,0.025600001215934753
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,256,1,16,1,0.02611200014750163
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,256,1,32,1,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,256,1,64,1,0.024746666351954143
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,128,1,4,131071,28.227925618489582
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,128,1,2,131071,28.23577626546224
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,256,1,1,1,0.05205333232879639
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,256,1,2,1,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,256,1,4,1,0.02679466704527537
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,256,1,8,1,0.0264533335963885
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,256,1,16,1,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,128,1,16,131071,28.140032450358074
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,256,1,32,1,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,256,1,64,1,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,128,1,8,131071,28.159317016601562
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,256,1,2,3,0.02918400118748347
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,256,1,1,3,0.05376000205675761
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,256,1,8,3,0.026965332527955372
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,256,1,16,3,0.025600001215934753
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,128,1,64,131071,28.136276245117188
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,256,1,4,3,0.0266239990790685
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,256,1,32,3,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,256,1,64,3,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,128,1,32,131071,28.14259084065755
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,256,1,1,3,0.05461333195368449
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,256,1,2,3,0.029525332152843475
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,256,1,4,3,0.027647999425729115
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,256,1,16,3,0.025429333249727886
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,256,1,32,3,0.0264533335963885
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,256,1,8,3,0.02611200014750163
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,256,1,2,7,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,256,1,64,3,0.02628266563018163
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,256,1,1,7,0.052906667192777
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,256,1,4,7,0.027818667391935985
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,256,1,8,7,0.025770666698614757
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,256,1,16,7,0.025429333249727886
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,256,1,32,7,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,256,1,64,7,0.02611200014750163
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,256,1,2,7,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,256,1,1,7,0.05256533126036326
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,256,1,4,7,0.02628266563018163
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,256,1,8,7,0.02611200014750163
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,256,1,16,7,0.025429333249727886
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,256,1,64,7,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,256,1,32,7,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,256,1,2,15,0.02935466667016347
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,256,1,1,15,0.054101333022117615
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,256,1,4,15,0.0266239990790685
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,256,1,8,15,0.025941332181294758
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,256,1,16,15,0.0264533335963885
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,256,1,32,15,0.025600001215934753
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,256,1,64,15,0.025600001215934753
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,256,1,1,15,0.053930665055910744
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,256,1,2,15,0.02867199977238973
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,256,1,32,15,0.025429333249727886
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,256,1,8,15,0.025941332181294758
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,256,1,16,15,0.025600001215934753
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,256,1,4,15,0.0264533335963885
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,256,1,64,15,0.025258667767047882
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,256,1,2,31,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,256,1,1,31,0.053930665055910744
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,256,1,4,31,0.027477333943049114
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,256,1,8,31,0.027477333943049114
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,256,1,32,31,0.025770666698614757
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,256,1,16,31,0.025770666698614757
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,256,1,1,31,0.053077335158983864
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,256,1,2,31,0.029696000119050343
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,256,1,4,31,0.027477333943049114
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,256,1,64,31,0.025429333249727886
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,256,1,8,31,0.027136000494162243
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,256,1,16,31,0.0266239990790685
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,256,1,32,31,0.025258667767047882
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,256,1,64,31,0.02628266563018163
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,256,1,2,63,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,256,1,4,63,0.027818667391935985
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,256,1,1,63,0.05597866574923197
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,256,1,8,63,0.0264533335963885
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,256,1,16,63,0.027306665976842243
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,256,1,32,63,0.026965332527955372
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,256,1,64,63,0.02628266563018163
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,256,1,1,63,0.056320001681645714
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,256,1,2,63,0.030207999050617218
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,256,1,4,63,0.027306665976842243
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,256,1,32,63,0.02628266563018163
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,256,1,8,63,0.0266239990790685
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,256,1,64,63,0.025770666698614757
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,256,1,16,63,0.02611200014750163
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,256,1,1,127,0.16435199975967407
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,256,1,2,127,0.05034666756788889
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,256,1,4,127,0.04693333307902018
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,256,1,8,127,0.04727466901143392
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,256,1,16,127,0.04625066618124644
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,256,1,64,127,0.045909335215886436
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,256,1,2,127,0.05034666756788889
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,256,1,4,127,0.048298666874567665
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,256,1,8,127,0.04744533201058706
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,256,1,1,127,0.17015467087427774
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,256,1,16,127,0.046762665112813316
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,256,1,32,127,0.04625066618124644
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,256,1,64,127,0.04659200211366018
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,256,1,32,127,0.046762665112813316
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,256,1,1,255,0.25514666239420575
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,256,1,4,255,0.08925867080688477
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,256,1,2,255,0.14028799533843994
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,256,1,16,255,0.08601599931716919
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,256,1,32,255,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,256,1,8,255,0.08806399504343669
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,256,1,64,255,0.08618666728337605
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,256,1,1,255,0.25941334168116253
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,256,1,2,255,0.1493333379427592
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,256,1,4,255,0.08823466300964355
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,256,1,8,255,0.08584533135096233
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,256,1,16,255,0.08584533135096233
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,256,1,32,255,0.08584533135096233
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,256,1,1,511,0.4055039882659912
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,256,1,2,511,0.2979840040206909
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,256,1,64,255,0.08533333738644917
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,256,1,4,511,0.26641066869099933
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,256,1,8,511,0.25463465849558514
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,256,1,32,511,0.24200532833735147
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,256,1,16,511,0.24541866779327393
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,256,1,1,511,0.4055039882659912
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,256,1,64,511,0.24200532833735147
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,256,1,2,511,0.29678932825724286
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,256,1,8,511,0.25412267446517944
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,256,1,16,511,0.2486613392829895
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,256,1,4,511,0.26641066869099933
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,256,1,32,511,0.24337067206700644
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,256,1,64,511,0.24217599630355835
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,256,1,1,1023,0.7410346666971842
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,256,1,2,1023,0.5258239905039469
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,256,1,4,1023,0.49612800280253094
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,256,1,8,1023,0.4819626808166504
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,256,1,16,1023,0.47308798631032306
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,256,1,64,1023,0.4643839995066325
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,256,1,1,1023,0.7357439994812012
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,256,1,32,1023,0.46745598316192627
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,256,1,2,1023,0.5254826545715332
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,256,1,4,1023,0.4935679833094279
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,256,1,8,1023,0.4797439972559611
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,256,1,16,1023,0.47086934248606366
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,256,1,32,1023,0.4689919948577881
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,256,1,64,1023,0.4631893237431844
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,256,1,4,2047,0.9376426537831625
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,256,1,1,2047,1.4074880282084148
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,256,1,2,2047,0.9611946741739908
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,256,1,8,2047,0.9180160363515218
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,256,1,16,2047,0.9077760378519694
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,256,1,32,2047,0.9016319910685221
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,256,1,64,2047,0.8978772958119711
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,256,1,1,2047,1.4112426439921062
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,256,1,2,2047,0.9661440054575602
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,256,1,4,2047,0.9378133614857992
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,256,1,16,2047,0.9076053301493326
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,256,1,32,2047,0.9029973347981771
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,256,1,64,2047,0.9233067035675049
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,256,1,8,2047,0.920746644337972
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,256,1,1,4095,2.7180372873942056
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,256,1,2,4095,1.8462719917297363
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,256,1,4,4095,1.8148694038391113
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,256,1,8,4095,1.8027520179748535
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,256,1,16,4095,1.793706734975179
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,256,1,32,4095,1.7860266367594402
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,256,1,64,4095,1.7810773849487305
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,256,1,1,4095,2.5390079816182456
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,256,1,2,4095,1.848149299621582
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,256,1,4,4095,1.8191359837849934
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,256,1,8,4095,1.8012159665425618
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,256,1,16,4095,1.7890987396240234
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,256,1,32,4095,1.7832959493001301
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,256,1,64,4095,1.780394713083903
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,256,1,1,8191,5.1418453852335615
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,256,1,8,8191,3.567445437113444
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,256,1,4,8191,3.5838292439778647
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,256,1,16,8191,3.557546615600586
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,256,1,32,8191,3.5490134557088218
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,256,1,2,8191,3.6109654108683267
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,256,1,64,8191,3.5476481119791665
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,256,1,1,8191,5.220863978068034
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,256,1,2,8191,3.6104532877604165
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,256,1,4,8191,3.587242762247721
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,256,1,8,8191,3.5652265548706055
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,256,1,16,8191,3.5517441431681314
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,256,1,32,8191,3.5486720403035483
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,256,1,64,8191,3.5462827682495117
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,256,1,2,16383,7.122261047363281
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,256,1,1,16383,10.315434773763021
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,256,1,4,16383,7.08403205871582
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,256,1,8,16383,7.068159739176433
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,256,1,16,16383,7.047509511311849
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,256,1,32,16383,7.048362731933594
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,256,1,64,16383,7.037781397501628
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,256,1,1,16383,10.278570810953775
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,256,1,2,16383,7.1145814259847
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,256,1,4,16383,7.091541290283203
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,256,1,8,16383,7.072938919067383
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,256,1,16,16383,7.048533121744792
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,256,1,32,16383,7.028906504313151
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,256,1,64,16383,7.030783971150716
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,256,1,1,32767,20.75716272989909
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,256,1,4,32767,14.075733184814453
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,256,1,8,32767,14.048938751220703
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,256,1,16,32767,14.026580810546875
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,256,1,2,32767,14.099967956542969
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,256,1,64,32767,14.048085530598959
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,256,1,32,32767,14.03835678100586
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,256,1,1,32767,20.6923090616862
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,256,1,2,32767,14.117546081542969
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,256,1,4,32767,14.059349060058594
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,256,1,16,32767,14.032213846842447
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,256,1,8,32767,14.055936177571615
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,256,1,64,32767,14.019755045572916
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,256,1,32,32767,14.028799692789713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,256,1,1,65535,42.46050008138021
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,256,1,2,65535,28.12859853108724
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,256,1,4,65535,28.046847025553387
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,256,1,32,65535,27.99974314371745
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,256,1,8,65535,28.027903238932293
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,256,1,16,65535,27.992576599121094
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,256,1,64,65535,28.010325113932293
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,512,1,1,1,0.2730666597684224
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,256,1,1,65535,41.947306315104164
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,512,1,2,1,0.05171200136343638
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,512,1,8,1,0.0481279989083608
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,512,1,4,1,0.04966400067011515
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,256,1,2,65535,28.085589090983074
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,512,1,16,1,0.04625066618124644
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,512,1,32,1,0.04539733131726583
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,256,1,4,65535,28.070912679036457
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,512,1,64,1,0.04693333307902018
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,256,1,8,65535,28.027732849121094
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,512,1,1,1,0.27153066794077557
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,512,1,2,1,0.051882664362589516
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,512,1,4,1,0.04983466863632202
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,512,1,8,1,0.04693333307902018
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,512,1,16,1,0.04693333307902018
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,512,1,64,1,0.04488533238569895
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,512,1,32,1,0.04539733131726583
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,512,1,1,3,0.27460267146428424
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,512,1,2,3,0.052906667192777
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,512,1,4,3,0.048810665806134544
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,512,1,8,3,0.04693333307902018
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,256,1,16,65535,28.01715087890625
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,512,1,16,3,0.04625066618124644
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,256,1,32,65535,27.987454732259113
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,512,1,64,3,0.04693333307902018
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,512,1,32,3,0.045567999283472695
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,256,1,64,65535,27.994283040364582
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,512,1,1,3,0.25702399015426636
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,512,1,2,3,0.051541333397229515
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,512,1,8,3,0.04778666794300079
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,512,1,4,3,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,512,1,16,3,0.046762665112813316
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,512,1,32,3,0.045226668318112694
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,512,1,64,3,0.04607999821503957
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,512,1,1,7,0.26282666126887005
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,512,1,8,7,0.04863999783992767
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,512,1,2,7,0.054272000988324486
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,512,1,4,7,0.05171200136343638
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,512,1,32,7,0.045567999283472695
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,512,1,16,7,0.04693333307902018
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,512,1,64,7,0.04539733131726583
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,512,1,1,7,0.2621440092722575
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,512,1,4,7,0.05034666756788889
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,512,1,2,7,0.052906667192777
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,512,1,8,7,0.04761599997679392
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,512,1,32,7,0.04642133414745331
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,512,1,16,7,0.048298666874567665
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,512,1,64,7,0.04625066618124644
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,512,1,1,15,0.27153066794077557
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,512,1,2,15,0.054101333022117615
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,512,1,4,15,0.04949333270390829
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,512,1,32,15,0.045738667249679565
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,512,1,16,15,0.05017599960168203
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,512,1,8,15,0.048810665806134544
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,512,1,64,15,0.04727466901143392
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,512,1,1,15,0.2706773281097412
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,512,1,2,15,0.05376000205675761
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,512,1,4,15,0.0481279989083608
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,512,1,8,15,0.04778666794300079
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,512,1,16,15,0.048469334840774536
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,512,1,32,15,0.04898133377234141
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,512,1,64,15,0.04642133414745331
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,512,1,1,31,0.2862079938252767
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,512,1,2,31,0.0773119976123174
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,512,1,4,31,0.06519466638565063
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,512,1,16,31,0.04949333270390829
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,512,1,32,31,0.04761599997679392
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,512,1,8,31,0.04761599997679392
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,512,1,64,31,0.049322664737701416
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,512,1,1,31,0.28859732548395794
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,512,1,2,31,0.05853866537412008
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,512,1,8,31,0.04761599997679392
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,512,1,4,31,0.04949333270390829
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,512,1,16,31,0.04778666794300079
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,512,1,32,31,0.04642133414745331
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,512,1,64,31,0.04744533201058706
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,512,1,1,63,0.3155626654624939
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,512,1,2,63,0.14967466394106546
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,512,1,4,63,0.05034666756788889
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,512,1,8,63,0.04898133377234141
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,512,1,16,63,0.05034666756788889
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,512,1,32,63,0.04778666794300079
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,512,1,64,63,0.04863999783992767
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,512,1,1,63,0.3114666740099589
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,512,1,4,63,0.05034666756788889
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,512,1,2,63,0.1565013329188029
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,512,1,8,63,0.05120000243186951
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,512,1,32,63,0.04949333270390829
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,512,1,16,63,0.049322664737701416
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,512,1,64,63,0.04727466901143392
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,512,1,1,127,0.36642134189605713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,512,1,2,127,0.23381332556406656
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,512,1,8,127,0.09045333663622539
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,512,1,4,127,0.1527466674645742
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,512,1,32,127,0.0885759989420573
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,512,1,16,127,0.09130666653315227
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,512,1,64,127,0.0865280032157898
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,512,1,1,127,0.36744534969329834
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,512,1,2,127,0.23398399353027344
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,512,1,4,127,0.1367039978504181
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,512,1,8,127,0.09113599856694539
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,512,1,16,127,0.08806399504343669
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,512,1,32,127,0.08823466300964355
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,512,1,64,127,0.0890880028406779
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,512,1,1,255,0.5053439935048422
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,512,1,2,255,0.3628373146057129
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,512,1,4,255,0.30139732360839844
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,512,1,8,255,0.27409066756566364
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,512,1,16,255,0.2595840096473694
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,512,1,32,255,0.2513920068740845
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,512,1,64,255,0.24661332368850708
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,512,1,4,255,0.3015679915746053
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,512,1,1,255,0.5070506731669108
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,512,1,8,255,0.27426133553187054
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,512,1,2,255,0.36300798257191974
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,512,1,16,255,0.25924267371495563
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,512,1,32,255,0.2513920068740845
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,512,1,64,255,0.24712532758712769
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,512,1,1,511,0.7932586669921875
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,512,1,4,511,0.5285546779632568
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,512,1,16,511,0.4853760004043579
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,512,1,2,511,0.5898240009943644
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,512,1,8,511,0.5003946622212728
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,512,1,32,511,0.47547733783721924
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,512,1,64,511,0.47138134638468426
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,512,1,1,511,0.8198826313018799
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,512,1,2,511,0.5901653369267782
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,512,1,4,511,0.5288960138956705
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,512,1,16,511,0.48264535268147785
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,512,1,8,511,0.5003946622212728
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,512,1,32,511,0.4771840174992879
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,512,1,64,511,0.46984533468882245
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,512,1,1,1023,1.4095360438028972
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,512,1,2,1023,1.0301439762115479
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,512,1,4,1023,0.9705813725789388
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,512,1,8,1023,0.936789353688558
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,512,1,32,1023,0.9081172943115234
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,512,1,16,1023,0.9200639724731445
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,512,1,64,1023,0.9029973347981771
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,512,1,1,1023,1.3970774014790852
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,512,1,2,1023,1.0289493401845295
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,512,1,4,1023,0.9675093491872152
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,512,1,8,1023,0.9374720255533854
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,512,1,16,1023,0.9190399646759033
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,512,1,32,1023,0.9081172943115234
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,512,1,64,1023,0.9028266270955404
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,512,1,1,2047,2.486954689025879
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,512,1,2,2047,1.9106133778889973
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,512,1,4,2047,1.8413227399190266
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,512,1,8,2047,1.8106026649475098
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,512,1,16,2047,1.7908053398132324
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,512,1,32,2047,1.7783466974894206
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,512,1,64,2047,1.7725440661112468
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,512,1,1,2047,2.485077381134033
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,512,1,2,2047,1.90993070602417
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,512,1,4,2047,1.8619732856750488
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,512,1,8,2047,1.8123092651367188
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,512,1,16,2047,1.7894399960835774
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,512,1,32,2047,1.7773226102193196
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,512,1,64,2047,1.7704960505167644
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,512,1,1,4095,4.994730631510417
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,512,1,2,4095,3.661482810974121
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,512,1,4,4095,3.5845120747884116
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,512,1,8,4095,3.5572052001953125
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,512,1,16,4095,3.535872141520182
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,512,1,32,4095,3.5179519653320312
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,512,1,64,4095,3.5102720260620117
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,512,1,1,4095,4.9805653889973955
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,512,1,2,4095,3.653631846110026
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,512,1,4,4095,3.5903145472208657
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,512,1,8,4095,3.5549866358439126
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,512,1,16,4095,3.5350186030069985
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,512,1,32,4095,3.5211947758992515
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,512,1,64,4095,3.5083945592244468
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,512,1,1,8191,9.950208028157553
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,512,1,2,8191,7.16424560546875
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,512,1,4,8191,7.089663823445638
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,512,1,8,8191,7.046314875284831
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,512,1,16,8191,7.019349416097005
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,512,1,32,8191,7.001258850097656
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,512,1,64,8191,6.992554982503255
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,512,1,1,8191,10.097834904988607
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,512,1,2,8191,7.1707305908203125
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,512,1,4,8191,7.094783782958984
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,512,1,8,8191,7.059797286987305
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,512,1,16,8191,7.017130533854167
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,512,1,32,8191,7.000064214070638
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,512,1,64,8191,6.9922135670979815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,512,1,2,16383,14.15509287516276
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,512,1,1,16383,20.14549382527669
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,512,1,4,16383,14.074026743570963
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,512,1,8,16383,13.985621134440104
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,512,1,16,16383,13.958997090657553
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,512,1,32,16383,13.936640421549479
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,512,1,64,16383,13.93271509806315
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,512,1,1,16383,19.887444814046223
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,512,1,4,16383,14.08017094930013
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,512,1,2,16383,14.137685139973959
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,512,1,16,16383,13.970261891682943
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,512,1,8,16383,13.983573913574219
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,512,1,64,16383,13.926400502522787
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,512,1,32,16383,13.939200083414713
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,512,1,1,32767,41.40902455647787
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,512,1,2,32767,28.138837178548176
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,512,1,4,32767,27.989334106445312
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,512,1,8,32767,27.9017817179362
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,512,1,16,32767,27.824127197265625
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,512,1,32,32767,27.83897654215495
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,512,1,1,32767,40.81152089436849
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,512,1,64,32767,27.80688985188802
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,1024,1,1,1,0.5865813493728638
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1024,1,2,1,0.25565866629282635
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1024,1,4,1,0.08789333701133728
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1024,1,8,1,0.09011200070381165
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1024,1,16,1,0.08413867155710857
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1024,1,32,1,0.08550399541854858
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1024,1,64,1,0.08038400113582611
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,1024,1,1,1,0.5570559899012247
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1024,1,2,1,0.2573653260866801
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1024,1,4,1,0.09198932846387227
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,512,1,2,32767,28.11272430419922
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1024,1,8,1,0.08430932958920796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1024,1,16,1,0.0885759989420573
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,512,1,4,32767,27.990699768066406
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1024,1,64,1,0.08430932958920796
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1024,1,32,1,0.08055466910203297
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,1024,1,1,3,0.5894826650619507
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,512,1,8,32767,27.905535380045574
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1024,1,4,3,0.09471999605496724
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1024,1,8,3,0.08533333738644917
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1024,1,2,3,0.2792106668154399
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1024,1,16,3,0.08499200145403545
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,512,1,16,32767,27.86798858642578
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,512,1,64,32767,27.795455932617188
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1024,1,32,3,0.08669867118199666
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1024,1,64,3,0.08260266482830048
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,512,1,32,32767,27.816446940104168
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1024,1,4,3,0.09011200070381165
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,1024,1,1,3,0.5713920195897421
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1024,1,2,3,0.28177066644032794
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1024,1,16,3,0.08584533135096233
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1024,1,8,3,0.09096533060073853
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1024,1,32,3,0.0817493349313736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1024,1,64,3,0.08243200182914734
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,1024,1,1,7,0.5889706611633301
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1024,1,4,7,0.0942080020904541
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1024,1,8,7,0.09147733449935913
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1024,1,2,7,0.2908160090446472
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1024,1,16,7,0.08721066514650981
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1024,1,64,7,0.0817493349313736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1024,1,32,7,0.08738133311271667
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,1024,1,1,7,0.5707093477249146
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1024,1,2,7,0.2943999965985616
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1024,1,4,7,0.0897706647713979
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1024,1,8,7,0.08994133273760478
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1024,1,16,7,0.08772266904513042
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1024,1,32,7,0.08721066514650981
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1024,1,64,7,0.0817493349313736
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,1024,1,1,15,0.5838506619135538
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1024,1,2,15,0.30702932675679523
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1024,1,8,15,0.0865280032157898
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1024,1,4,15,0.11622400085131328
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1024,1,16,15,0.09011200070381165
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1024,1,64,15,0.08567466338475545
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1024,1,32,15,0.08772266904513042
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,1024,1,1,15,0.5858986775080363
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1024,1,4,15,0.09386666615804036
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1024,1,2,15,0.30003199974695843
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1024,1,8,15,0.08669867118199666
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1024,1,16,15,0.08669867118199666
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1024,1,32,15,0.08840533097585042
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1024,1,64,15,0.0897706647713979
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,1024,1,1,31,0.6145706574122111
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1024,1,2,31,0.3447466691335042
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1024,1,4,31,0.1520639955997467
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1024,1,8,31,0.09318400422732036
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1024,1,16,31,0.08721066514650981
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1024,1,32,31,0.08516266942024231
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1024,1,64,31,0.08959999680519104
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,1024,1,1,31,0.614741325378418
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1024,1,2,31,0.340992013613383
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1024,1,4,31,0.16605866948763529
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1024,1,8,31,0.08755200107892354
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1024,1,16,31,0.09011200070381165
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1024,1,32,31,0.08601599931716919
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1024,1,64,31,0.08567466338475545
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,1024,1,1,63,0.6432426770528158
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1024,1,2,63,0.3691519896189372
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1024,1,4,63,0.237226665019989
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1024,1,8,63,0.14148267110188803
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1024,1,16,63,0.09318400422732036
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1024,1,32,63,0.09096533060073853
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1024,1,64,63,0.08942932883898418
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,1024,1,1,63,0.6543360153834025
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1024,1,2,63,0.408405343691508
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1024,1,4,63,0.23705599705378214
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1024,1,32,63,0.0885759989420573
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1024,1,8,63,0.16913066307703653
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1024,1,16,63,0.09267200032869975
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1024,1,64,63,0.09096533060073853
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,1024,1,1,127,0.7437653541564941
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1024,1,2,127,0.47598934173583984
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1024,1,4,127,0.35703468322753906
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1024,1,8,127,0.29627732435862225
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1024,1,16,127,0.26794666051864624
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1024,1,32,127,0.2542933424313863
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1024,1,64,127,0.24593067169189453
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,1024,1,1,127,0.7446186542510986
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1024,1,4,127,0.3548159996668498
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1024,1,2,127,0.4766720136006673
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1024,1,8,127,0.29730133215586346
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1024,1,32,127,0.25412267446517944
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1024,1,16,127,0.2696533401807149
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1024,1,64,127,0.2476373314857483
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,1024,1,1,255,0.9780906836191813
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1024,1,2,255,0.719701369603475
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1024,1,4,255,0.59989333152771
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1024,1,8,255,0.539136012395223
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1024,1,16,255,0.512170672416687
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1024,1,32,255,0.49715201059977215
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1024,1,64,255,0.4884479840596517
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,1024,1,1,255,0.9811626275380453
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1024,1,2,255,0.7237973213195801
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1024,1,4,255,0.5975040197372437
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1024,1,8,255,0.537770668665568
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1024,1,16,255,0.5114880005518595
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1024,1,32,255,0.4945919911066691
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1024,1,64,255,0.4867413441340129
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,1024,1,1,511,1.5324160257975261
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1024,1,2,511,1.1683839956919353
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1024,1,4,511,1.0550613403320312
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1024,1,8,511,1.0219519933064778
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1024,1,16,511,0.9623893102010092
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1024,1,64,511,0.9320106506347656
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1024,1,32,511,0.943615992863973
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,1024,1,1,511,1.5281492869059246
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1024,1,2,511,1.1805013020833333
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1024,1,4,511,1.0581333637237549
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1024,1,8,511,0.9978880087534586
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1024,1,16,511,0.9637546539306641
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1024,1,32,511,0.9400320053100586
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1024,1,64,511,0.934058666229248
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,1024,1,1,1023,2.630143960316976
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1024,1,2,1023,2.0834986368815103
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1024,1,4,1023,1.966250737508138
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1024,1,8,1023,1.9008852640787761
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1024,1,16,1023,1.8614613215128581
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1024,1,32,1023,1.837056001027425
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1024,1,64,1023,1.8240853945414226
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,1024,1,1,1023,2.6263893445332847
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1024,1,4,1023,1.9602773984273274
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1024,1,2,1023,2.1015893618265786
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1024,1,8,1023,1.897813320159912
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1024,1,16,1023,1.8566826184590657
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1024,1,64,1023,1.8276693026224773
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1024,1,32,1023,1.834496021270752
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,1024,1,1,2047,5.024938583374023
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1024,1,2,2047,3.867648124694824
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1024,1,4,2047,3.767807960510254
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1024,1,8,2047,3.6805973052978516
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1024,1,16,2047,3.641514778137207
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1024,1,32,2047,3.61028258005778
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1024,1,64,2047,3.593045234680176
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,1024,1,1,2047,4.896597226460774
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1024,1,2,2047,3.884373346964518
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1024,1,4,2047,3.74783992767334
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1024,1,8,2047,3.6792319615681968
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1024,1,16,2047,3.644927978515625
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1024,1,32,2047,3.6104532877604165
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1024,1,64,2047,3.597653388977051
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1024,1,2,4095,7.439189275105794
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,1024,1,1,4095,10.271914800008139
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1024,1,4,4095,7.323306401570638
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1024,1,8,4095,7.224831899007161
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1024,1,16,4095,7.195648193359375
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1024,1,32,4095,7.163562774658203
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1024,1,64,4095,7.150421142578125
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,1024,1,1,4095,9.91539192199707
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1024,1,2,4095,7.453354517618815
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1024,1,4,4095,7.270058949788411
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1024,1,16,4095,7.185920079549153
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1024,1,8,4095,7.222784042358398
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1024,1,32,4095,7.151786804199219
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1024,1,64,4095,7.150421142578125
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1024,1,2,8191,14.524927775065104
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,1024,1,1,8191,20.339370727539062
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1024,1,4,8191,14.390271504720053
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1024,1,8,8191,14.325247446695963
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1024,1,16,8191,14.255616505940756
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1024,1,64,8191,14.208340962727865
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1024,1,32,8191,14.259712219238281
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,1024,1,1,8191,19.54901377360026
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1024,1,2,8191,14.480042775472006
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1024,1,4,8191,14.40768051147461
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1024,1,16,8191,14.249642690022787
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1024,1,8,8191,14.339584350585938
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1024,1,32,8191,14.217727661132812
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1024,1,64,8191,14.196224212646484
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,128,1024,1,1,16383,39.93753560384115
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,64,1024,1,2,16383,28.817237854003906
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,32,1024,1,4,16383,28.529151916503906
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,16,1024,1,8,16383,28.390912373860676
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,8,1024,1,16,16383,28.306602478027344
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,4,1024,1,32,16383,28.402005513509113
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,fp8,2,1024,1,64,16383,28.29858144124349
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,128,1024,1,1,16383,40.16401163736979
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,64,1024,1,2,16383,28.705279032389324
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,32,1024,1,4,16383,28.57591501871745
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,16,1024,1,8,16383,28.399614969889324
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,8,1024,1,16,16383,28.32281494140625
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,2,1024,1,64,16383,28.350123087565105
TRTLLM,1.0.0,NVIDIA L40S,mla_generation,default,float16,float16,4,1024,1,32,16383,28.311721801757812
