framework,version,device,op_name,kernel_source,mla_dtype,kv_cache_dtype,num_heads,batch_size,isl,tp_size,step,latency
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,1,16,1,0,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,1,16,2,0,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,1,16,4,0,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,1,16,8,0,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,1,16,16,0,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,1,16,32,0,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,1,16,64,0,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,1,16,2,0,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,1,16,4,0,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,1,16,1,0,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,1,16,32,0,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,1,16,64,0,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,1,16,8,0,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,1,32,16,0,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,1,32,1,0,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,1,16,16,0,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,1,32,32,0,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,1,32,64,0,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,1,32,2,0,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,1,32,8,0,0.011221333096424738
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,1,32,4,0,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,1,32,2,0,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,1,32,1,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,1,32,4,0,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,1,32,8,0,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,1,32,64,0,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,1,32,32,0,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,1,64,1,0,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,1,64,2,0,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,1,64,4,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,1,32,16,0,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,1,64,8,0,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,1,64,16,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,1,64,32,0,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,1,64,2,0,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,1,64,64,0,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,1,64,4,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,1,64,8,0,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,1,64,32,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,1,64,16,0,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,1,64,1,0,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,1,64,64,0,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,1,128,1,0,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,1,128,4,0,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,1,128,8,0,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,1,128,2,0,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,1,128,32,0,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,1,128,16,0,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,1,128,1,0,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,1,128,2,0,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,1,128,4,0,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,1,128,8,0,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,1,128,16,0,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,1,128,64,0,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,1,128,32,0,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,1,128,64,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,1,256,1,0,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,1,256,2,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,1,256,4,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,1,256,8,0,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,1,256,16,0,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,1,256,64,0,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,1,256,32,0,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,1,256,1,0,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,1,256,4,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,1,256,2,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,1,256,8,0,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,1,256,16,0,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,1,256,32,0,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,1,512,1,0,0.05905066430568695
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,1,512,2,0,0.03583999971548716
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,1,512,4,0,0.022357332209746044
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,1,512,16,0,0.019968000551064808
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,1,512,32,0,0.020138667275508244
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,1,512,64,0,0.019797333826621372
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,1,256,64,0,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,1,512,1,0,0.05870933334032694
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,1,512,2,0,0.03549866626660029
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,1,512,8,0,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,1,512,4,0,0.022357332209746044
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,1,512,8,0,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,1,512,16,0,0.019968000551064808
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,1,512,32,0,0.019968000551064808
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,1,1024,1,0,0.16947199900945029
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,1,1024,2,0,0.10069333513577779
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,1,1024,16,0,0.034304000437259674
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,1,1024,8,0,0.03618133316437403
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,1,1024,4,0,0.05836800237496694
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,1,1024,32,0,0.03362133353948593
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,1,512,64,0,0.019626667102177937
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,1,1024,64,0,0.03328000009059906
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,1,1024,1,0,0.16913066307703653
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,1,1024,2,0,0.10001066327095032
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,1,1024,4,0,0.05853866537412008
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,1,1024,16,0,0.0341333324710528
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,1,1024,8,0,0.03618133316437403
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,1,1024,32,0,0.03362133353948593
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,1,1536,2,0,0.18449066082636514
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,1,1536,1,0,0.37700267632802326
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,1,1024,64,0,0.03310933212439219
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,1,1536,4,0,0.11212799946467082
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,1,1536,16,0,0.048469334840774536
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,1,1536,8,0,0.06724266707897186
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,1,1536,64,0,0.04659200211366018
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,1,1536,32,0,0.04761599997679392
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,1,1536,4,0,0.11349333326021831
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,1,1536,2,0,0.1879040002822876
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,1,1536,8,0,0.06741333504517873
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,1,1536,1,0,0.37307735284169513
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,1,1536,32,0,0.04761599997679392
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,1,1536,16,0,0.048469334840774536
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,1,1536,64,0,0.046762665112813316
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,1,2048,1,0,0.6104746659596761
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,1,2048,2,0,0.30054400364557904
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,1,2048,4,0,0.17885865767796835
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,1,2048,8,0,0.11502933502197266
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,1,2048,32,0,0.06092800199985504
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,1,2048,64,0,0.059903999169667564
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,1,2048,2,0,0.29627732435862225
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,1,2048,4,0,0.17885865767796835
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,1,2048,1,0,0.6137173175811768
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,1,2048,8,0,0.11571199695269267
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,1,2048,16,0,0.0628053347269694
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,1,2048,32,0,0.06092800199985504
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,1,2048,16,0,0.06263466676076253
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,1,2048,64,0,0.06007466713587443
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,1,3072,1,0,1.2810239791870117
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,1,3072,4,0,0.33109333117802936
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,1,3072,8,0,0.2034346659978231
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,1,3072,2,0,0.6638933420181274
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,1,3072,16,0,0.130730668703715
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,1,3072,32,0,0.0890880028406779
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,1,3072,1,0,1.295360008875529
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,1,3072,64,0,0.08721066514650981
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,1,3072,4,0,0.3298986752827962
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,1,3072,2,0,0.6488746802012125
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,1,3072,8,0,0.2034346659978231
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,1,3072,32,0,0.08874666690826416
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,1,3072,16,0,0.13209600249926248
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,1,4096,2,0,1.1059199968973796
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,1,4096,1,0,2.2068907419840493
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,1,4096,4,0,0.5415253241856893
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,1,4096,8,0,0.3114666740099589
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,1,4096,16,0,0.20360533396402994
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,1,4096,32,0,0.1204906702041626
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,1,3072,64,0,0.08755200107892354
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,1,4096,64,0,0.11400533715883891
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,1,4096,1,0,2.2266880671183267
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,1,4096,4,0,0.5406719843546549
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,1,4096,2,0,1.1037013530731201
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,1,4096,8,0,0.3114666740099589
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,1,4096,16,0,0.20360533396402994
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,1,4096,64,0,0.11400533715883891
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,1,4096,32,0,0.12032000223795573
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,1,6144,1,0,5.021354675292969
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,1,6144,8,0,0.6074026823043823
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,1,6144,4,0,1.193130652109782
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,1,6144,32,0,0.23466666539510092
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,1,6144,16,0,0.36027733484903973
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,1,6144,2,0,2.3528107007344565
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,1,6144,1,0,4.750165303548177
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,1,6144,2,0,2.377386728922526
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,1,6144,64,0,0.17100799083709717
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,1,6144,8,0,0.6089386542638143
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,1,6144,32,0,0.23278933763504028
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,1,6144,16,0,0.3638613224029541
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,1,6144,64,0,0.1704960068066915
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,1,6144,4,0,1.1987626552581787
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,1,8192,2,0,4.201301256815593
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,1,8192,4,0,2.1090985933939614
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,1,8192,16,0,0.5756586790084839
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,1,8192,8,0,1.0330453713734944
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,1,8192,1,0,9.299968083699545
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,1,8192,32,0,0.3662506739298503
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,1,8192,64,0,0.23552000522613525
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,1,8192,2,0,4.329471906026204
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,1,8192,4,0,2.0913492838541665
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,1,8192,1,0,8.636245091756185
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,1,8192,8,0,1.0313386917114258
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,1,8192,16,0,0.5761706829071045
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,1,8192,64,0,0.23569067319234213
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,1,8192,32,0,0.36522666613260907
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,1,10240,8,0,1.5853226979573567
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,1,10240,2,0,6.972245534261067
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,1,10240,4,0,3.17354679107666
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,1,10240,1,0,13.8874880472819
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,1,10240,32,0,0.5114880005518595
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,1,10240,16,0,0.839680035909017
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,1,10240,64,0,0.3298986752827962
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,1,10240,4,0,3.186858812967936
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,1,10240,8,0,1.645055929819743
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,1,10240,32,0,0.5104639927546183
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,1,10240,1,0,14.298795064290365
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,1,10240,16,0,0.8458240032196045
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,1,10240,64,0,0.32631466786066693
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,1,10240,2,0,6.679040273030599
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,1,12288,1,0,19.772074381510418
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,1,12288,16,0,1.1774293581644695
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,1,12288,64,0,0.4389546712239583
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,1,12288,32,0,0.6799360116322836
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,1,12288,8,0,2.2355626424153647
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,1,12288,2,0,9.896618525187174
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,1,12288,4,0,4.701866785685222
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,1,12288,8,0,2.2289066314697266
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,1,12288,4,0,4.814847946166992
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,1,12288,2,0,9.968981424967447
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,1,12288,16,0,1.1683839956919353
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,1,12288,32,0,0.6802773475646973
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,1,12288,1,0,19.353429158528645
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,1,12288,64,0,0.44305066267649335
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,1,16384,8,0,3.94649600982666
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,1,16384,16,0,2.0276907285054526
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,1,16384,32,0,1.0861226717631023
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,1,16384,2,0,16.884564717610676
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,1,16384,64,0,0.6714026927947998
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,1,16384,4,0,8.686421076456705
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,1,16384,1,0,34.51289621988932
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,1,16384,8,0,4.071765263875325
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,1,16384,16,0,2.0181333223978677
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,1,16384,32,0,1.0890239874521892
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,1,16384,4,0,8.264874776204428
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,2,16,1,0,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,1,16384,2,0,17.27402623494466
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,1,16384,1,0,34.12445831298828
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,1,16384,64,0,0.6664533217748007
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,2,16,2,0,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,2,16,4,0,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,2,16,8,0,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,2,16,16,0,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,2,16,32,0,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,2,16,2,0,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,2,16,8,0,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,2,16,4,0,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,2,16,1,0,0.012106666962305704
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,2,16,16,0,0.011066666493813196
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,2,16,64,0,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,2,16,32,0,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,2,16,64,0,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,2,32,1,0,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,2,32,4,0,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,2,32,8,0,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,2,32,2,0,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,2,32,32,0,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,2,32,16,0,0.011253333340088526
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,2,32,64,0,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,2,32,2,0,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,2,32,8,0,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,2,32,1,0,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,2,32,4,0,0.011418666690587997
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,2,32,16,0,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,2,32,32,0,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,2,32,64,0,0.011258666714032492
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,2,64,1,0,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,2,64,2,0,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,2,64,16,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,2,64,4,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,2,64,32,0,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,2,64,64,0,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,2,64,1,0,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,2,64,2,0,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,2,64,8,0,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,2,64,4,0,0.01138666644692421
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,2,64,8,0,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,2,64,16,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,2,64,64,0,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,2,64,32,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,2,128,1,0,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,2,128,2,0,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,2,128,8,0,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,2,128,16,0,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,2,128,64,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,2,128,32,0,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,2,128,1,0,0.01911466692884763
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,2,128,4,0,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,2,128,4,0,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,2,128,2,0,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,2,128,8,0,0.011909333368142446
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,2,128,16,0,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,2,128,64,0,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,2,128,32,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,2,256,1,0,0.039936001102129616
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,2,256,8,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,2,256,4,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,2,256,16,0,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,2,256,32,0,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,2,256,64,0,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,2,256,2,0,0.025258667767047882
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,2,256,1,0,0.03976533313592275
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,2,256,2,0,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,2,256,4,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,2,256,16,0,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,2,256,64,0,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,2,256,8,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,2,256,32,0,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,2,512,2,0,0.059903999169667564
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,2,512,8,0,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,2,512,16,0,0.021162666380405426
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,2,512,32,0,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,2,512,1,0,0.10410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,2,512,64,0,0.020138667275508244
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,2,512,4,0,0.03601066768169403
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,2,512,1,0,0.10205866893132527
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,2,512,2,0,0.059562668204307556
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,2,512,4,0,0.03583999971548716
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,2,512,8,0,0.022698665658632915
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,2,512,16,0,0.021162666380405426
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,2,512,64,0,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,2,1024,1,0,0.38809601465861004
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,2,1024,2,0,0.17169066270192465
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,2,512,32,0,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,2,1024,8,0,0.05922133227189382
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,2,1024,4,0,0.1013759970664978
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,2,1024,16,0,0.0365226666132609
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,2,1024,32,0,0.034815999368826546
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,2,1024,64,0,0.0341333324710528
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,2,1024,2,0,0.17083734273910522
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,2,1024,1,0,0.38758401075998944
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,2,1024,16,0,0.03669333209594091
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,2,1024,8,0,0.05922133227189382
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,2,1024,32,0,0.034815999368826546
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,2,1024,64,0,0.0341333324710528
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,2,1024,4,0,0.10171733299891154
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,2,1536,2,0,0.3729066848754883
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,2,1536,1,0,0.7453013261159261
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,2,1536,4,0,0.18500266472498575
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,2,1536,8,0,0.11315199732780457
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,2,1536,16,0,0.06758399804433186
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,2,1536,32,0,0.049322664737701416
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,2,1536,2,0,0.37034666538238525
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,2,1536,4,0,0.18756266434987387
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,2,1536,1,0,0.7746559778849283
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,2,1536,64,0,0.0481279989083608
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,2,1536,8,0,0.11025066177050273
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,2,1536,16,0,0.0682666649421056
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,2,1536,32,0,0.04915200173854828
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,2,1536,64,0,0.048298666874567665
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,2,2048,1,0,1.2595199743906658
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,2,2048,2,0,0.6166186730066935
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,2,2048,4,0,0.30395734310150146
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,2,2048,16,0,0.11315199732780457
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,2,2048,32,0,0.06348800162474315
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,2,2048,64,0,0.06195199986298879
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,2,2048,1,0,1.280512015024821
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,2,2048,2,0,0.6157653331756592
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,2,2048,8,0,0.1800533334414164
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,2,2048,4,0,0.3015679915746053
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,2,2048,16,0,0.11383466919263203
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,2,2048,32,0,0.06365866462389629
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,2,2048,8,0,0.18056533734003702
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,2,2048,64,0,0.06178133189678192
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,2,3072,2,0,1.300821304321289
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,2,3072,4,0,0.668842633565267
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,2,3072,1,0,2.713770548502604
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,2,3072,8,0,0.33348266283671063
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,2,3072,16,0,0.20172800620396933
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,2,3072,32,0,0.13346133629480997
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,2,3072,1,0,2.6540373166402182
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,2,3072,2,0,1.3013333479563396
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,2,3072,64,0,0.09011200070381165
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,2,3072,16,0,0.20309333006540933
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,2,3072,4,0,0.6696960131327311
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,2,3072,32,0,0.13158399860064188
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,2,3072,8,0,0.33399466673533124
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,2,3072,64,0,0.08994133273760478
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,2,4096,2,0,2.2420479456583657
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,2,4096,1,0,4.690773328145345
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,2,4096,8,0,0.5589333375295004
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,2,4096,16,0,0.3165866732597351
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,2,4096,64,0,0.1225386659304301
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,2,4096,32,0,0.20241065820058188
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,2,4096,4,0,1.1356159845987956
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,2,4096,1,0,4.770133336385091
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,2,4096,2,0,2.2220800717671714
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,2,4096,4,0,1.1144533157348633
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,2,4096,8,0,0.560640017191569
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,2,4096,16,0,0.31641600529352826
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,2,4096,32,0,0.20411733786265054
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,2,4096,64,0,0.12236799796422322
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,2,6144,2,0,5.025792121887207
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,2,6144,16,0,0.6145706574122111
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,2,6144,4,0,2.3688534100850425
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,2,6144,8,0,1.2054186662038167
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,2,6144,32,0,0.366592009862264
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,2,6144,64,0,0.2362026572227478
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,2,6144,1,0,10.482688268025717
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,2,6144,1,0,10.518186569213867
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,2,6144,4,0,2.364415963490804
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,2,6144,2,0,5.2268374760945635
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,2,6144,32,0,0.3657386700312297
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,2,6144,16,0,0.6164480050404867
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,2,6144,8,0,1.19705597559611
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,2,6144,64,0,0.2362026572227478
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,2,8192,2,0,8.78114128112793
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,2,8192,4,0,4.130304018656413
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,2,8192,8,0,2.1386240323384604
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,2,8192,16,0,1.0929493109385173
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,2,8192,64,0,0.3676160176595052
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,2,8192,32,0,0.5806080102920532
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,2,8192,1,0,18.320383707682293
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,2,8192,8,0,2.14408540725708
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,2,8192,2,0,8.68403180440267
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,2,8192,32,0,0.5799253384272257
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,2,8192,4,0,4.150442759195964
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,2,8192,1,0,18.48473612467448
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,2,8192,16,0,1.0859519640604656
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,2,8192,64,0,0.3671040137608846
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,2,10240,16,0,1.6423254013061523
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,2,10240,4,0,6.796629587809245
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,2,10240,8,0,3.1578451792399087
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,2,10240,64,0,0.5164373318354288
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,2,10240,2,0,14.155946095784506
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,2,10240,32,0,0.9111893177032471
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,2,10240,1,0,27.866111755371094
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,2,10240,4,0,6.630741119384766
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,2,10240,8,0,3.1105705897013345
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,2,10240,16,0,1.6541013717651367
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,2,10240,1,0,28.035242716471355
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,2,10240,64,0,0.5166079998016357
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,2,10240,32,0,0.889685312906901
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,2,10240,2,0,13.92196273803711
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,2,12288,4,0,9.760085423787435
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,2,12288,32,0,1.271125316619873
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,2,12288,8,0,4.609706560770671
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,2,12288,64,0,0.6918826897939047
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,2,12288,16,0,2.3111680348714194
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,2,12288,2,0,19.444564819335938
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,2,12288,8,0,4.702549298604329
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,2,12288,16,0,2.383018652598063
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,2,12288,64,0,0.6860799789428711
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,2,12288,1,0,39.97491200764974
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,2,12288,32,0,1.2479146321614583
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,2,12288,4,0,10.172245025634766
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,2,12288,2,0,19.79647954305013
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,2,12288,1,0,39.60763804117838
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,2,16384,32,0,2.0807679494222007
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,2,16384,64,0,1.1989333629608154
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,2,16384,16,0,4.020394643147786
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,2,16384,4,0,16.95402654012044
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,2,16384,8,0,8.562005360921225
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,2,16384,2,0,34.65335591634115
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,2,16384,16,0,4.082175890604655
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,2,16384,4,0,17.000277201334637
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,2,16384,64,0,1.2584959665934246
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,2,16384,8,0,8.829952239990234
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,2,16384,32,0,2.1449386278788247
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,4,16,1,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,2,16384,1,0,71.24650573730469
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,2,16384,2,0,34.18606821695963
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,4,16,2,0,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,4,16,4,0,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,2,16384,1,0,70.33395385742188
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,4,16,32,0,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,4,16,64,0,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,4,16,1,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,4,16,2,0,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,4,16,4,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,4,16,16,0,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,4,16,8,0,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,4,16,8,0,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,4,16,16,0,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,4,16,32,0,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,4,16,64,0,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,4,32,1,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,4,32,2,0,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,4,32,16,0,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,4,32,64,0,0.011087999989589056
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,4,32,1,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,4,32,4,0,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,4,32,8,0,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,4,32,2,0,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,4,32,4,0,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,4,32,32,0,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,4,32,16,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,4,32,8,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,4,32,32,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,4,64,4,0,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,4,64,2,0,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,4,32,64,0,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,4,64,16,0,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,4,64,1,0,0.01791999985774358
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,4,64,32,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,4,64,64,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,4,64,8,0,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,4,64,1,0,0.018090666582187016
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,4,64,2,0,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,4,64,8,0,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,4,64,64,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,4,64,32,0,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,4,64,4,0,0.01191466674208641
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,4,128,2,0,0.019285333653291065
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,4,128,4,0,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,4,128,8,0,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,4,64,16,0,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,4,128,1,0,0.0315733328461647
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,4,128,16,0,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,4,128,32,0,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,4,128,1,0,0.0315733328461647
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,4,128,2,0,0.01911466692884763
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,4,128,4,0,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,4,128,64,0,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,4,128,32,0,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,4,128,64,0,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,4,128,8,0,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,4,256,1,0,0.06997333467006683
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,4,256,2,0,0.040106666584809623
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,4,128,16,0,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,4,256,4,0,0.025600001215934753
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,4,256,16,0,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,4,256,32,0,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,4,256,1,0,0.07099733253320058
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,4,256,8,0,0.01621333385507266
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,4,256,2,0,0.039936001102129616
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,4,256,64,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,4,256,8,0,0.01621333385507266
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,4,256,16,0,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,4,256,32,0,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,4,256,64,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,4,512,2,0,0.10393599669138591
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,4,256,4,0,0.025770666698614757
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,4,512,8,0,0.036864000062147774
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,4,512,4,0,0.0602453351020813
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,4,512,16,0,0.023552000522613525
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,4,512,1,0,0.3275093237559001
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,4,512,32,0,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,4,512,64,0,0.020992000897725422
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,4,512,1,0,0.3278506596883138
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,4,512,2,0,0.10461866855621338
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,4,512,4,0,0.059903999169667564
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,4,512,8,0,0.03669333209594091
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,4,512,16,0,0.023552000522613525
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,4,512,64,0,0.020992000897725422
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,4,1024,2,0,0.39372801780700684
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,4,1024,1,0,0.8166399796803793
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,4,1024,4,0,0.1730560064315796
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,4,512,32,0,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,4,1024,64,0,0.03618133316437403
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,4,1024,16,0,0.06092800199985504
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,4,1024,1,0,0.8113493124643961
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,4,1024,32,0,0.03788800040880839
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,4,1024,2,0,0.39867734909057617
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,4,1024,8,0,0.10308266679445903
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,4,1024,4,0,0.17066667477289835
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,4,1024,16,0,0.06075733403364817
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,4,1024,64,0,0.03601066768169403
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,4,1024,32,0,0.03754666695992152
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,4,1024,8,0,0.10222933689753215
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,4,1536,1,0,1.5827627182006836
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,4,1536,8,0,0.1858560045560201
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,4,1536,16,0,0.11400533715883891
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,4,1536,2,0,0.7879679997762045
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,4,1536,32,0,0.06860800087451935
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,4,1536,64,0,0.0506879985332489
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,4,1536,4,0,0.3834880193074544
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,4,1536,2,0,0.7615146636962891
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,4,1536,4,0,0.37085866928100586
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,4,1536,8,0,0.18995199600855509
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,4,1536,1,0,1.609557310740153
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,4,1536,32,0,0.06946133573849995
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,4,1536,64,0,0.050517335534095764
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,4,2048,2,0,1.2716373602549236
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,4,1536,16,0,0.11520000298817952
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,4,2048,4,0,0.6234453519185384
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,4,2048,8,0,0.3413333495457967
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,4,2048,32,0,0.11690666278203328
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,4,2048,16,0,0.18158932526906332
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,4,2048,1,0,2.649258613586426
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,4,2048,64,0,0.06553600231806438
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,4,2048,1,0,2.6094932556152344
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,4,2048,16,0,0.17988266547520956
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,4,2048,4,0,0.6191786527633667
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,4,2048,8,0,0.3155626654624939
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,4,2048,2,0,1.314304033915202
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,4,2048,64,0,0.06570666531721751
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,4,2048,32,0,0.11724799871444702
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,4,3072,2,0,2.660010655721029
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,4,3072,4,0,1.3267626762390137
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,4,3072,16,0,0.33894399801890057
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,4,3072,8,0,0.6661119858423868
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,4,3072,1,0,5.860181172688802
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,4,3072,64,0,0.1367039978504181
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,4,3072,2,0,2.674858729044596
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,4,3072,32,0,0.2070186734199524
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,4,3072,16,0,0.33740798632303876
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,4,3072,8,0,0.6743040084838867
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,4,3072,4,0,1.3269333044687908
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,4,3072,64,0,0.1372160017490387
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,4,3072,1,0,5.710506439208984
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,4,3072,32,0,0.20309333006540933
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,4,4096,4,0,2.226858615875244
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,4,4096,1,0,9.928533554077148
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,4,4096,8,0,1.1422719955444336
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,4,4096,32,0,0.3203413287798564
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,4,4096,16,0,0.599722663561503
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,4,4096,64,0,0.21026132504145303
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,4,4096,2,0,4.816213289896647
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,4,4096,16,0,0.6033066511154175
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,4,4096,4,0,2.2570667266845703
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,4,4096,2,0,4.604586601257324
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,4,4096,8,0,1.145685354868571
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,4,4096,32,0,0.3184640010197957
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,4,4096,64,0,0.20838399728139242
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,4,4096,1,0,10.152448018391928
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,4,6144,2,0,10.946219126383463
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,4,6144,4,0,5.06385072072347
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,4,6144,16,0,1.261738697687785
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,4,6144,8,0,2.3997440338134766
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,4,6144,64,0,0.373418649037679
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,4,6144,32,0,0.697002649307251
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,4,6144,1,0,21.664596557617188
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,4,6144,2,0,10.62826665242513
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,4,6144,4,0,4.942847887674968
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,4,6144,16,0,1.2426239649454753
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,4,6144,32,0,0.7074133555094401
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,4,6144,8,0,2.4325119654337564
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,4,6144,64,0,0.370688001314799
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,4,6144,1,0,21.584554036458332
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,4,8192,8,0,4.301994641621907
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,4,8192,16,0,2.1992106437683105
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,4,8192,4,0,9.2313601175944
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,4,8192,64,0,0.6766933600107828
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,4,8192,32,0,1.160533348719279
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,4,8192,2,0,18.238975524902344
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,4,8192,1,0,37.057535807291664
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,4,8192,64,0,0.6884693304697672
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,4,8192,8,0,4.318037350972493
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,4,8192,32,0,1.164629300435384
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,4,8192,16,0,2.1553492546081543
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,4,8192,4,0,9.326762517293295
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,8,16,1,0,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,4,8192,2,0,18.151935577392578
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,8,16,2,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,4,8192,1,0,37.82792409261068
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,8,16,8,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,8,16,32,0,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,8,16,4,0,0.01209066684047381
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,8,16,16,0,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,8,16,64,0,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,8,16,1,0,0.024405332903067272
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,8,16,2,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,8,16,4,0,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,8,16,8,0,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,8,16,64,0,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,8,16,16,0,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,8,32,1,0,0.025941332181294758
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,8,16,32,0,0.010911999891201654
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,8,32,4,0,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,8,32,16,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,8,32,2,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,8,32,8,0,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,8,32,32,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,8,32,64,0,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,8,32,2,0,0.01621333385507266
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,8,32,1,0,0.025941332181294758
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,8,32,8,0,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,8,32,4,0,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,8,32,16,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,8,32,32,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,8,32,64,0,0.011402666568756104
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,8,64,1,0,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,8,64,2,0,0.01826133330663045
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,8,64,4,0,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,8,64,8,0,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,8,64,16,0,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,8,64,32,0,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,8,64,1,0,0.029525332152843475
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,8,64,2,0,0.01791999985774358
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,8,64,8,0,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,8,64,4,0,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,8,64,16,0,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,8,64,64,0,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,8,64,32,0,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,8,64,64,0,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,8,128,1,0,0.053247998158137
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,8,128,2,0,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,8,128,4,0,0.019797333826621372
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,8,128,16,0,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,8,128,64,0,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,8,128,8,0,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,8,128,1,0,0.05341866612434387
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,8,128,32,0,0.012256000190973282
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,8,128,2,0,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,8,128,4,0,0.019797333826621372
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,8,128,8,0,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,8,128,16,0,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,8,128,32,0,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,8,128,64,0,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,8,256,1,0,0.3141973416010539
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,8,256,4,0,0.040618665516376495
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,8,256,32,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,8,256,8,0,0.0266239990790685
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,8,256,16,0,0.016895999511082966
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,8,256,64,0,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,8,256,2,0,0.07133866846561432
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,8,256,1,0,0.3131733338038127
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,8,256,2,0,0.0721919983625412
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,8,256,4,0,0.040618665516376495
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,8,256,8,0,0.0264533335963885
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,8,256,32,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,8,256,16,0,0.0170666662355264
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,8,256,64,0,0.014848000059525171
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,8,512,2,0,0.3362133502960205
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,8,512,4,0,0.10752000411351521
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,8,512,8,0,0.060415998101234436
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,8,512,16,0,0.03788800040880839
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,8,512,32,0,0.024746666351954143
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,8,512,1,0,0.7338666915893555
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,8,512,64,0,0.023039999107519787
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,8,512,2,0,0.33740798632303876
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,8,512,1,0,0.731818675994873
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,8,512,4,0,0.1053013304869334
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,8,512,16,0,0.037717332442601524
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,8,512,8,0,0.06092800199985504
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,8,512,32,0,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,8,1024,2,0,0.8290987014770508
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,8,1024,1,0,1.707690715789795
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,8,1024,4,0,0.4077226718266805
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,8,1024,8,0,0.18568533658981323
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,8,512,64,0,0.023039999107519787
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,8,1024,16,0,0.10376532872517903
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,8,1024,64,0,0.039936001102129616
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,8,1024,32,0,0.06195199986298879
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,8,1024,2,0,0.8340480327606201
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,8,1024,1,0,1.6981333096822102
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,8,1024,4,0,0.4041386842727661
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,8,1024,8,0,0.1868799924850464
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,8,1024,16,0,0.10222933689753215
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,8,1024,64,0,0.039936001102129616
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,8,1024,32,0,0.06263466676076253
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,8,1536,1,0,3.239765485127767
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,8,1536,2,0,1.6291839281717937
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,8,1536,16,0,0.1904639999071757
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,8,1536,8,0,0.39321601390838623
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,8,1536,4,0,0.793941338857015
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,8,1536,64,0,0.08294400076071422
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,8,1536,32,0,0.11520000298817952
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,8,1536,2,0,1.6030720074971516
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,8,1536,1,0,3.2588799794514975
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,8,1536,4,0,0.8040106296539307
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,8,1536,32,0,0.1160533328851064
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,8,1536,8,0,0.3867306709289551
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,8,1536,16,0,0.1962666710217794
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,8,1536,64,0,0.08226133386294048
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,8,2048,2,0,2.644480069478353
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,8,2048,4,0,1.3221546808878581
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,8,2048,8,0,0.6729386647542318
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,8,2048,1,0,5.592405319213867
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,8,2048,16,0,0.35140268007914227
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,8,2048,64,0,0.1220266620318095
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,8,2048,32,0,0.1802240014076233
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,8,2048,4,0,1.3245440324147542
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,8,2048,1,0,5.540010452270508
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,8,2048,2,0,2.655061403910319
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,8,2048,8,0,0.6732800006866455
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,8,2048,32,0,0.18227199713389078
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,8,2048,16,0,0.3561813433965047
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,8,2048,64,0,0.12185600399971008
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,8,3072,8,0,1.3847893079121907
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,8,3072,16,0,0.7342080275217692
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,8,3072,4,0,2.8388694127400718
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,8,3072,32,0,0.40994131565093994
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,8,3072,1,0,11.843925476074219
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,8,3072,2,0,5.700778961181641
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,8,3072,64,0,0.20940800507863364
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,8,3072,1,0,12.061866760253906
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,8,3072,4,0,2.777258555094401
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,8,3072,2,0,5.690026601155599
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,8,3072,16,0,0.717141310373942
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,8,3072,8,0,1.430357297261556
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,8,3072,64,0,0.21691733598709106
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,8,3072,32,0,0.41915734608968097
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,8,4096,2,0,10.126165390014648
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,8,4096,1,0,20.155733744303387
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,8,4096,8,0,2.3381333351135254
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,8,4096,4,0,4.7457278569539385
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,8,4096,16,0,1.2134400208791096
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,8,4096,32,0,0.687445322672526
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,8,4096,64,0,0.40994131565093994
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,8,4096,2,0,9.986047744750977
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,8,4096,4,0,4.988757451375325
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,8,4096,8,0,2.327722708384196
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,8,4096,32,0,0.6683306694030762
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,8,4096,16,0,1.2470613320668538
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,8,4096,64,0,0.4154026508331299
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,16,16,1,0,0.04215466479460398
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,8,4096,1,0,20.439552307128906
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,16,16,2,0,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,16,16,4,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,16,16,8,0,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,16,16,64,0,0.011039999624093374
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,16,16,16,0,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,16,16,1,0,0.04181333382924398
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,16,16,32,0,0.011946666985750198
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,16,16,4,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,16,16,16,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,16,16,8,0,0.012448000411192576
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,16,16,2,0,0.024746666351954143
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,16,16,32,0,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,16,16,64,0,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,16,32,1,0,0.045226668318112694
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,16,32,2,0,0.02628266563018163
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,16,32,4,0,0.01621333385507266
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,16,32,8,0,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,16,32,16,0,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,16,32,64,0,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,16,32,1,0,0.04488533238569895
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,16,32,8,0,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,16,32,4,0,0.016384000579516094
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,16,32,2,0,0.02628266563018163
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,16,32,32,0,0.011567999919255575
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,16,32,16,0,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,16,32,32,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,16,32,64,0,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,16,64,1,0,0.05085866649945577
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,16,64,2,0,0.029696000119050343
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,16,64,8,0,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,16,64,4,0,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,16,64,32,0,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,16,64,16,0,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,16,64,1,0,0.051541333397229515
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,16,64,2,0,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,16,64,4,0,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,16,64,8,0,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,16,64,16,0,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,16,64,64,0,0.011946666985750198
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,16,64,32,0,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,16,128,1,0,0.3126613299051921
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,16,128,2,0,0.0554666668176651
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,16,128,4,0,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,16,128,8,0,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,16,128,16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,16,128,32,0,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,16,128,64,0,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,16,64,64,0,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,16,128,1,0,0.3123199939727783
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,16,128,2,0,0.05376000205675761
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,16,128,8,0,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,16,128,4,0,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,16,128,16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,16,128,32,0,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,16,128,64,0,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,16,256,2,0,0.32972800731658936
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,16,256,4,0,0.07150933146476746
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,16,256,16,0,0.027306665976842243
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,16,256,8,0,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,16,256,32,0,0.01826133330663045
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,16,256,64,0,0.016895999511082966
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,16,256,1,0,0.733354647954305
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,16,256,1,0,0.7304533322652181
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,16,256,2,0,0.3290453354517619
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,16,256,4,0,0.0718506673971812
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,16,256,16,0,0.027306665976842243
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,16,256,8,0,0.04147200038035711
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,16,256,32,0,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,16,256,64,0,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,16,512,1,0,1.5115946133931477
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,16,512,4,0,0.3534506559371948
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,16,512,8,0,0.12339199582735698
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,16,512,2,0,0.7529813448588053
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,16,512,32,0,0.03942399968703588
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,16,512,16,0,0.06229333579540253
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,16,512,64,0,0.026965332527955372
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,16,512,1,0,1.5109119415283203
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,16,512,4,0,0.353279987970988
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,16,512,2,0,0.7540053526560465
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,16,512,8,0,0.12288000186284383
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,16,512,32,0,0.03925333420435587
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,16,512,64,0,0.02679466704527537
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,16,1024,1,0,3.4761387507120767
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,16,512,16,0,0.06297599772612254
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,16,1024,4,0,0.8688639799753824
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,16,1024,8,0,0.43485867977142334
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,16,1024,2,0,1.7452373504638672
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,16,1024,32,0,0.10854400197664897
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,16,1024,16,0,0.2290346622467041
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,16,1024,64,0,0.07492266595363617
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,16,1024,1,0,3.474090576171875
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,16,1024,4,0,0.8690346876780192
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,16,1024,8,0,0.4333226680755615
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,16,1024,2,0,1.723903973897298
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,16,1024,64,0,0.06621866424878438
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,16,1024,32,0,0.10905599594116211
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,16,1024,16,0,0.23603200912475586
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,16,1536,2,0,3.40445868174235
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,16,1536,1,0,6.701226552327474
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,16,1536,4,0,1.710762659708659
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,16,1536,8,0,0.8393386999766032
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,16,1536,64,0,0.12424533565839131
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,16,1536,32,0,0.25890133778254193
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,16,1536,16,0,0.44305066267649335
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,16,1536,4,0,1.6802132924397786
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,16,1536,2,0,3.3937066396077475
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,16,1536,1,0,6.7539628346761065
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,16,1536,16,0,0.43434667587280273
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,16,1536,8,0,0.8710827032725016
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,16,1536,32,0,0.26077866554260254
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,16,1536,64,0,0.12151466806729634
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,16,2048,2,0,5.37821896870931
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,16,2048,4,0,2.788522720336914
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,16,2048,8,0,1.4347947438557942
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,16,2048,16,0,0.7372799714406332
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,16,2048,64,0,0.2723840077718099
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,16,2048,32,0,0.4242773453394572
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,16,2048,1,0,11.133098602294922
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,16,2048,1,0,11.478357950846354
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,16,2048,2,0,5.548373540242513
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,16,2048,4,0,2.780501365661621
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,16,2048,8,0,1.4119253158569336
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,16,2048,16,0,0.733184019724528
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,16,2048,32,0,0.4212053219477336
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,16,2048,64,0,0.2723840077718099
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,32,16,2,0,0.04266666869322459
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,32,16,1,0,0.0747519979874293
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,32,16,4,0,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,32,16,8,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,32,16,16,0,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,32,16,32,0,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,32,16,64,0,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,32,16,1,0,0.07441066702206929
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,32,16,2,0,0.04232533276081085
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,32,16,8,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,32,16,16,0,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,32,16,4,0,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,32,16,32,0,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,32,16,64,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,32,32,4,0,0.0264533335963885
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,32,32,2,0,0.045909335215886436
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,32,32,1,0,0.08004266520341237
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,32,32,8,0,0.016895999511082966
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,32,32,16,0,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,32,32,64,0,0.012063999970753988
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,32,32,32,0,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,32,32,1,0,0.08021333316961925
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,32,32,2,0,0.045909335215886436
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,32,32,4,0,0.026965332527955372
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,32,32,8,0,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,32,32,16,0,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,32,32,32,0,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,32,32,64,0,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,32,64,1,0,0.3037866751352946
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,32,64,2,0,0.051029334465662636
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,32,64,4,0,0.030207999050617218
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,32,64,8,0,0.0194560003777345
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,32,64,32,0,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,32,64,64,0,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,32,64,1,0,0.3017386595408122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,32,64,16,0,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,32,64,4,0,0.030207999050617218
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,32,64,2,0,0.051541333397229515
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,32,64,8,0,0.019285333653291065
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,32,64,16,0,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,32,64,32,0,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,32,64,64,0,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,32,128,1,0,0.736255963643392
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,32,128,2,0,0.32494932413101196
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,32,128,4,0,0.05563733478387197
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,32,128,16,0,0.021503999829292297
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,32,128,32,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,32,128,64,0,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,32,128,8,0,0.03345066557327906
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,32,128,1,0,0.7360853354136149
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,32,128,2,0,0.32443734010060626
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,32,128,4,0,0.056320001681645714
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,32,128,8,0,0.03310933212439219
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,32,128,16,0,0.021503999829292297
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,32,128,32,0,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,32,128,64,0,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,32,256,2,0,0.7519573370615641
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,32,256,4,0,0.34986666838328045
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,32,256,8,0,0.09181867043177287
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,32,256,1,0,1.4917972882588704
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,32,256,16,0,0.043178667624791466
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,32,256,32,0,0.029525332152843475
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,32,256,64,0,0.019968000551064808
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,32,256,1,0,1.493333339691162
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,32,256,2,0,0.7540053526560465
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,32,256,4,0,0.34815998872121173
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,32,256,8,0,0.09437867005666097
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,32,256,32,0,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,32,256,64,0,0.020138667275508244
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,32,256,16,0,0.04334933559099833
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,32,512,1,0,3.0429865519205728
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,32,512,2,0,1.5419732729593914
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,32,512,4,0,0.7821653683980306
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,32,512,8,0,0.3979946772257487
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,32,512,16,0,0.1771519978841146
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,32,512,32,0,0.06604800124963124
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,32,512,64,0,0.04351999859015147
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,32,512,2,0,1.5426559448242188
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,32,512,4,0,0.78438401222229
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,32,512,8,0,0.39560532569885254
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,32,512,32,0,0.06553600231806438
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,32,512,16,0,0.176639993985494
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,32,512,1,0,3.0426454544067383
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,32,512,64,0,0.04351999859015147
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,32,1024,1,0,7.026175816853841
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,32,1024,2,0,3.5554987589518228
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,32,1024,8,0,0.9412266413370768
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,32,1024,4,0,1.787050724029541
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,32,1024,16,0,0.5034666856129965
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,32,1024,32,0,0.2996906638145447
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,32,1024,64,0,0.20292266209920248
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,32,1024,1,0,6.9853865305582685
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,32,1024,4,0,1.82152525583903
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,32,1024,8,0,0.9451519648234049
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,32,1024,32,0,0.29627732435862225
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,32,1024,16,0,0.5191680192947388
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,32,1024,2,0,3.5150505701700845
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,32,1024,64,0,0.1962666710217794
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,64,16,1,0,0.13960533340771994
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,64,16,4,0,0.04334933559099833
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,64,16,8,0,0.025429333249727886
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,64,16,2,0,0.07526400188604991
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,64,16,64,0,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,64,16,32,0,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,64,16,16,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,64,16,1,0,0.15052800377209982
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,64,16,8,0,0.025429333249727886
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,64,16,4,0,0.042837331692377724
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,64,16,2,0,0.07509333391984303
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,64,16,16,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,64,16,32,0,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,64,16,64,0,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,64,32,1,0,0.2988373239835103
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,64,32,2,0,0.08277333279450734
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,64,32,4,0,0.04642133414745331
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,64,32,8,0,0.027647999425729115
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,64,32,64,0,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,64,32,16,0,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,64,32,32,0,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,64,32,1,0,0.29730133215586346
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,64,32,4,0,0.046762665112813316
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,64,32,8,0,0.027647999425729115
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,64,32,16,0,0.017407999684413273
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,64,32,32,0,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,64,64,2,0,0.31692800919214886
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,64,32,2,0,0.08106666803359985
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,64,64,1,0,0.7505919933319092
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,64,64,4,0,0.054101333022117615
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,64,32,64,0,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,64,64,8,0,0.0314026673634847
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,64,64,16,0,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,64,64,32,0,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,64,64,64,0,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,64,64,4,0,0.05444266895453135
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,64,64,2,0,0.31573333342870075
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,64,64,16,0,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,64,64,1,0,0.754858652750651
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,64,64,32,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,64,64,8,0,0.0314026673634847
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,64,64,64,0,0.012800000607967377
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,64,128,1,0,1.499135971069336
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,64,128,4,0,0.3479893207550049
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,64,128,2,0,0.7531519730885824
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,64,128,8,0,0.08106666803359985
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,64,128,32,0,0.0240639994541804
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,64,128,1,0,1.5010132789611816
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,64,128,16,0,0.03549866626660029
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,64,128,64,0,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,64,128,2,0,0.754688024520874
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,64,128,4,0,0.34781865278879803
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,64,128,16,0,0.03532800078392029
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,64,128,8,0,0.08089600006739299
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,64,128,32,0,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,64,128,64,0,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,64,256,1,0,2.9858134587605796
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,64,256,4,0,0.7825067043304443
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,64,256,8,0,0.3991893529891968
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,64,256,16,0,0.17390932639439902
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,64,256,2,0,1.5278080304463704
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,64,256,64,0,0.03310933212439219
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,64,256,32,0,0.04744533201058706
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,64,256,1,0,2.980863889058431
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,64,256,2,0,1.5595520337422688
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,64,256,4,0,0.7835306326548258
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,64,256,8,0,0.39816534519195557
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,64,256,16,0,0.17271467049916586
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,64,256,64,0,0.03345066557327906
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,64,512,4,0,1.6356693903605144
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,64,512,2,0,3.087530771891276
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,64,256,32,0,0.04744533201058706
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,64,512,1,0,6.0660050710042315
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,64,512,8,0,0.8645973205566406
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,64,512,16,0,0.4940799872080485
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,64,512,32,0,0.2860373258590698
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,64,512,64,0,0.17339734236399332
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,64,512,1,0,6.0822188059488935
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,64,512,8,0,0.8644266923268636
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,64,512,16,0,0.4930560191472371
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,64,512,2,0,3.0837761561075845
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,64,512,32,0,0.28706133365631104
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,128,16,1,0,0.3386026620864868
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,64,512,64,0,0.16554666558901468
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,64,512,4,0,1.605631987253825
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,128,16,2,0,0.1437013347943624
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,128,16,8,0,0.04369066655635834
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,128,16,4,0,0.07679999868075053
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,128,16,16,0,0.0264533335963885
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,128,16,1,0,0.3386026620864868
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,128,16,64,0,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,128,16,4,0,0.07594666878382365
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,128,16,32,0,0.016895999511082966
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,128,16,2,0,0.14165332913398743
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,128,16,8,0,0.044031997521718345
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,128,16,16,0,0.02628266563018163
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,128,16,32,0,0.016895999511082966
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,128,16,64,0,0.011946666985750198
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,128,32,1,0,0.7234559853871664
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,128,32,2,0,0.31061333417892456
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,128,32,4,0,0.08396800359090169
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,128,32,16,0,0.02867199977238973
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,128,32,64,0,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,128,32,8,0,0.04778666794300079
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,128,32,32,0,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,128,32,1,0,0.7285760243733724
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,128,32,2,0,0.311296006043752
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,128,32,4,0,0.08567466338475545
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,128,32,8,0,0.04778666794300079
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,128,32,16,0,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,128,32,32,0,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,128,64,2,0,0.7698773543039957
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,128,64,1,0,1.5168852806091309
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,128,64,8,0,0.07645866771539052
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,128,64,4,0,0.3563520113627116
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,128,32,64,0,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,128,64,16,0,0.03362133353948593
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,128,64,32,0,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,128,64,64,0,0.016554666062196095
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,128,64,1,0,1.540437380472819
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,128,64,2,0,0.7703893184661865
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,128,64,4,0,0.3418453137079875
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,128,64,8,0,0.07560533285140991
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,128,64,64,0,0.016384000579516094
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,128,64,16,0,0.0339626669883728
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,128,64,32,0,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,128,128,1,0,2.996053377787272
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,128,128,4,0,0.7949653466542562
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,128,128,2,0,1.537706693013509
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,128,128,8,0,0.39953064918518066
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,128,128,16,0,0.1699840029080709
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,128,128,32,0,0.03976533313592275
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,128,128,2,0,1.5342933336893718
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,128,128,4,0,0.7927467028299967
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,128,128,1,0,2.9992958704630532
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,128,128,8,0,0.42444801330566406
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,128,128,64,0,0.027818667391935985
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,128,128,16,0,0.17066667477289835
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,128,128,32,0,0.03976533313592275
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,128,128,64,0,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,128,256,2,0,3.0452054341634116
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,128,256,8,0,0.8685226440429688
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,128,256,4,0,1.5941972732543945
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,128,256,32,0,0.2882560094197591
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,128,256,1,0,5.950634638468425
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,128,256,64,0,0.16025599837303162
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,128,256,16,0,0.4991999864578247
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,128,256,1,0,5.941077550252278
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,128,256,2,0,3.0504961013793945
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,128,256,16,0,0.49186134338378906
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,128,256,64,0,0.15923200050989786
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,128,256,32,0,0.3037866751352946
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,128,256,4,0,1.5916372934977214
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,128,256,8,0,0.8688639799753824
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,256,16,2,0,0.35942399501800537
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,256,16,4,0,0.14830933014551798
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,256,16,1,0,0.7355733712514242
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,256,16,8,0,0.077824001510938
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,256,16,16,0,0.045738667249679565
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,256,16,64,0,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,256,16,32,0,0.027647999425729115
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,256,16,1,0,0.7371093432108561
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,256,16,2,0,0.3476479848225911
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,256,16,4,0,0.14728533228238425
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,256,16,16,0,0.045567999283472695
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,256,16,8,0,0.07833600044250488
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,256,16,32,0,0.027818667391935985
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,256,16,64,0,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,256,32,2,0,0.7492266496022543
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,256,32,4,0,0.33501867453257245
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,256,32,8,0,0.09898666540781657
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,256,32,1,0,1.489749272664388
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,256,32,16,0,0.05034666756788889
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,256,32,64,0,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,256,32,1,0,1.4871892929077148
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,256,32,2,0,0.7550293604532877
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,256,32,32,0,0.0314026673634847
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,256,32,4,0,0.33587201436360675
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,256,32,8,0,0.0988159974416097
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,256,32,16,0,0.05000533163547516
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,256,32,32,0,0.031061333914597828
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,256,32,64,0,0.021503999829292297
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,256,64,2,0,1.5564799308776855
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,256,64,1,0,3.0156799952189126
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,256,64,8,0,0.400383989016215
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,256,64,4,0,0.8118613560994467
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,256,64,16,0,0.17169066270192465
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,256,64,32,0,0.038058665891488395
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,256,64,64,0,0.026965332527955372
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,256,64,2,0,1.561087926228841
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,256,64,1,0,3.0185813903808594
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,256,64,4,0,0.8103253046671549
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,256,64,8,0,0.4002133210500081
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,256,64,64,0,0.02679466704527537
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,256,64,32,0,0.03839999934037527
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,256,64,16,0,0.1728853384653727
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,256,128,1,0,5.969749450683594
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,256,128,2,0,3.062272071838379
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,256,128,8,0,0.8832000096638998
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,256,128,4,0,1.6083626747131348
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,256,128,16,0,0.5044906536738077
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,256,128,32,0,0.29661866029103595
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,1,256,128,64,0,0.15940266847610474
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,256,128,2,0,3.0600531895955405
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,256,128,1,0,5.972991943359375
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,256,128,4,0,1.6631466547648113
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,256,128,8,0,0.8777386347452799
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,256,128,32,0,0.2940586606661479
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,256,128,16,0,0.5038079818089803
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,1,16,1,0,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,1,16,2,0,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,1,256,128,64,0,0.15940266847610474
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,1,16,4,0,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,1,16,16,0,0.010890666395425797
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,1,16,8,0,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,1,16,32,0,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,1,16,64,0,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,1,16,2,0,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,1,16,1,0,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,1,16,16,0,0.01192533348997434
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,1,16,4,0,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,1,16,8,0,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,1,16,64,0,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,1,16,32,0,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,1,32,1,0,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,1,32,4,0,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,1,32,2,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,1,32,8,0,0.011578666667143503
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,1,32,16,0,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,1,32,32,0,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,1,32,1,0,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,1,32,2,0,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,1,32,64,0,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,1,32,4,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,1,32,16,0,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,1,32,8,0,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,1,32,64,0,0.01139733319481214
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,1,32,32,0,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,1,64,1,0,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,1,64,2,0,0.011946666985750198
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,1,64,4,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,1,64,8,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,1,64,16,0,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,1,64,32,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,1,64,64,0,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,1,64,1,0,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,1,64,2,0,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,1,64,4,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,1,64,8,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,1,64,16,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,1,64,32,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,1,64,64,0,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,1,128,1,0,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,1,128,8,0,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,1,128,4,0,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,1,128,2,0,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,1,128,32,0,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,1,128,16,0,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,1,128,64,0,0.011584000041087469
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,1,128,1,0,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,1,128,2,0,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,1,128,16,0,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,1,128,8,0,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,1,128,32,0,0.011770666887362799
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,1,128,4,0,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,1,256,1,0,0.03976533313592275
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,1,128,64,0,0.01108266661564509
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,1,256,2,0,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,1,256,4,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,1,256,8,0,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,1,256,16,0,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,1,256,32,0,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,1,256,64,0,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,1,256,2,0,0.025429333249727886
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,1,256,4,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,1,256,1,0,0.039936001102129616
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,1,256,8,0,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,1,256,16,0,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,1,256,32,0,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,1,256,64,0,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,1,512,2,0,0.059392000238100685
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,1,512,1,0,0.10564266641934712
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,1,512,8,0,0.022357332209746044
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,1,512,4,0,0.03549866626660029
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,1,512,16,0,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,1,512,32,0,0.020138667275508244
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,1,512,1,0,0.10222933689753215
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,1,512,2,0,0.05922133227189382
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,1,512,4,0,0.03566933423280716
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,1,512,8,0,0.02218666672706604
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,1,512,64,0,0.019968000551064808
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,1,512,16,0,0.020821332931518555
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,1,512,32,0,0.020138667275508244
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,1,512,64,0,0.019797333826621372
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,1,1024,2,0,0.17254400253295898
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,1,1024,1,0,0.3938986857732137
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,1,1024,4,0,0.10222933689753215
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,1,1024,8,0,0.05870933334032694
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,1,1024,64,0,0.03345066557327906
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,1,1024,16,0,0.03601066768169403
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,1,1024,32,0,0.034474665919939675
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,1,1024,1,0,0.3947519858678182
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,1,1024,2,0,0.17151999473571777
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,1,1024,4,0,0.09949866930643718
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,1,1024,8,0,0.05905066430568695
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,1,1024,16,0,0.03618133316437403
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,1,1024,32,0,0.034304000437259674
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,1,1024,64,0,0.03362133353948593
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,1,1536,1,0,0.754858652750651
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,1,1536,2,0,0.3717120091120402
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,1,1536,4,0,0.1884160041809082
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,1,1536,8,0,0.11076266566912334
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,1,1536,16,0,0.06946133573849995
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,1,1536,32,0,0.048469334840774536
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,1,1536,64,0,0.04744533201058706
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,1,1536,1,0,0.7511040369669596
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,1,1536,8,0,0.11281067132949829
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,1,1536,16,0,0.06724266707897186
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,1,1536,2,0,0.37887998421986896
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,1,1536,4,0,0.18705066045125326
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,1,1536,64,0,0.04761599997679392
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,1,1536,32,0,0.048298666874567665
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,1,2048,1,0,1.2491093476613362
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,1,2048,2,0,0.6157653331756592
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,1,2048,4,0,0.2991786599159241
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,1,2048,8,0,0.17578667402267456
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,1,2048,16,0,0.11639466881752014
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,1,2048,32,0,0.06263466676076253
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,1,2048,64,0,0.06075733403364817
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,1,2048,1,0,1.2967253526051838
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,1,2048,2,0,0.6128640174865723
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,1,2048,4,0,0.29764266808827716
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,1,2048,16,0,0.11366400122642517
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,1,2048,8,0,0.17783466974894205
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,1,2048,32,0,0.0628053347269694
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,1,2048,64,0,0.06075733403364817
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,1,3072,1,0,2.7127466201782227
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,1,3072,2,0,1.3347840309143066
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,1,3072,4,0,0.6459733247756958
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,1,3072,8,0,0.3304106593132019
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,1,3072,32,0,0.13192533453305563
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,1,3072,16,0,0.20104533433914185
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,1,3072,64,0,0.08874666690826416
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,1,3072,1,0,2.647552013397217
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,1,3072,2,0,1.328810691833496
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,1,3072,8,0,0.3298986752827962
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,1,3072,32,0,0.13192533453305563
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,1,3072,16,0,0.20036266247431436
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,1,3072,64,0,0.08891733487447102
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,1,4096,1,0,4.714666684468587
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,1,3072,4,0,0.6765226523081461
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,1,4096,4,0,1.1410773595174153
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,1,4096,2,0,2.2128639221191406
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,1,4096,8,0,0.5423786640167236
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,1,4096,32,0,0.20121600230534872
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,1,4096,16,0,0.3145386576652527
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,1,4096,64,0,0.1204906702041626
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,1,4096,2,0,2.27839994430542
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,1,4096,8,0,0.5469866593678793
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,1,4096,1,0,4.8411305745442705
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,1,4096,4,0,1.1298133532206218
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,1,4096,16,0,0.311296006043752
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,1,4096,32,0,0.20172800620396933
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,1,4096,64,0,0.12168533603350322
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,1,6144,1,0,10.889728546142578
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,1,6144,8,0,1.2030293146769206
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,1,6144,16,0,0.6138879855473837
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,1,6144,32,0,0.3597653309504191
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,1,6144,4,0,2.3632213274637857
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,1,6144,2,0,5.262165387471517
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,1,6144,64,0,0.2336426575978597
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,1,6144,2,0,4.967424074808757
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,1,6144,1,0,10.523818969726562
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,1,6144,4,0,2.370730717976888
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,1,6144,8,0,1.2248746554056804
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,1,6144,32,0,0.36300798257191974
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,1,6144,16,0,0.6075733502705892
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,1,6144,64,0,0.2397866646448771
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,1,8192,8,0,2.073941389719645
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,1,8192,4,0,4.34551461537679
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,1,8192,32,0,0.5768533150355021
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,1,8192,16,0,1.0586453278859456
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,1,8192,2,0,9.325738906860352
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,1,8192,64,0,0.36351998647054035
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,1,8192,1,0,18.332159678141277
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,1,8192,1,0,18.37004852294922
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,1,8192,8,0,2.0346879959106445
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,1,8192,4,0,4.283050537109375
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,1,8192,32,0,0.5889706611633301
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,1,8192,2,0,8.89804776509603
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,1,8192,16,0,1.0480639934539795
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,1,8192,64,0,0.36317865053812665
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,1,10240,1,0,27.97858174641927
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,1,10240,8,0,3.16979185740153
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,1,10240,2,0,14.19912592569987
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,1,10240,32,0,0.845141331354777
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,1,10240,64,0,0.5138773520787557
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,1,10240,16,0,1.6223573684692383
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,1,10240,4,0,6.9715627034505205
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,1,10240,1,0,27.759445190429688
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,1,10240,8,0,3.247957229614258
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,1,10240,2,0,13.948586781819662
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,1,10240,64,0,0.5135360161463419
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,1,10240,16,0,1.6133119265238445
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,1,10240,4,0,6.888618469238281
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,1,10240,32,0,0.846677303314209
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,1,12288,16,0,2.3323307037353516
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,1,12288,8,0,4.65937074025472
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,1,12288,4,0,9.983146667480469
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,1,12288,64,0,0.6814719835917155
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,1,12288,32,0,1.1709439754486084
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,1,12288,2,0,19.716607411702473
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,1,12288,1,0,39.63221232096354
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,1,12288,32,0,1.1723093191782634
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,1,12288,16,0,2.2543360392252603
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,1,12288,8,0,4.741973241170247
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,1,12288,4,0,9.919488271077475
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,1,12288,64,0,0.6869333585103353
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,1,12288,2,0,20.35268274943034
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,1,12288,1,0,39.540906270345054
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,1,16384,16,0,4.002133369445801
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,1,16384,8,0,8.578901290893555
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,1,16384,32,0,2.016256014506022
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,1,16384,64,0,1.098410685857137
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,1,16384,4,0,17.10318883260091
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,1,16384,2,0,34.27106221516927
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,1,16384,32,0,2.0174506505330405
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,1,16384,8,0,8.53879483540853
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,1,16384,16,0,4.119039853413899
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,1,16384,4,0,17.312768300374348
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,1,16384,1,0,71.52503458658855
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,1,16384,64,0,1.117184003194173
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,2,16,2,0,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,2,16,4,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,1,16384,2,0,34.51153055826823
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,2,16,1,0,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,2,16,8,0,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,2,16,16,0,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,2,16,32,0,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,1,16384,1,0,71.65047709147136
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,2,16,64,0,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,2,16,1,0,0.015189333508412043
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,2,16,4,0,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,2,16,8,0,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,2,16,2,0,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,2,16,16,0,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,2,16,32,0,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,2,16,64,0,0.011247999966144562
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,2,32,1,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,2,32,2,0,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,2,32,4,0,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,2,32,8,0,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,2,32,16,0,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,2,32,32,0,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,2,32,64,0,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,2,32,2,0,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,2,32,4,0,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,2,32,8,0,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,2,32,1,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,2,32,16,0,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,2,32,32,0,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,2,32,64,0,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,2,64,1,0,0.017749333133300144
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,2,64,2,0,0.01310933381319046
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,2,64,4,0,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,2,64,8,0,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,2,64,32,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,2,64,64,0,0.011578666667143503
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,2,64,2,0,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,2,64,1,0,0.01791999985774358
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,2,64,4,0,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,2,64,16,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,2,64,8,0,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,2,64,16,0,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,2,64,32,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,2,64,64,0,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,2,128,1,0,0.0315733328461647
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,2,128,2,0,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,2,128,4,0,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,2,128,8,0,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,2,128,16,0,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,2,128,64,0,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,2,128,1,0,0.0314026673634847
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,2,128,2,0,0.01911466692884763
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,2,128,32,0,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,2,128,4,0,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,2,128,8,0,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,2,128,16,0,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,2,128,32,0,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,2,128,64,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,2,256,1,0,0.06877866884072621
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,2,256,2,0,0.03976533313592275
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,2,256,8,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,2,256,16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,2,256,32,0,0.013823999712864557
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,2,256,64,0,0.013482666263977686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,2,256,1,0,0.07048533360163371
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,2,256,4,0,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,2,256,4,0,0.025258667767047882
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,2,256,2,0,0.039936001102129616
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,2,256,16,0,0.014165333161751429
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,2,256,8,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,2,256,32,0,0.013653332988421122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,2,512,1,0,0.33075199524561566
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,2,256,64,0,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,2,512,2,0,0.10410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,2,512,8,0,0.03601066768169403
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,2,512,16,0,0.03054933249950409
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,2,512,32,0,0.021162666380405426
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,2,512,64,0,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,2,512,1,0,0.3304106593132019
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,2,512,2,0,0.10376532872517903
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,2,512,4,0,0.059392000238100685
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,2,512,8,0,0.03583999971548716
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,2,512,16,0,0.022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,2,512,4,0,0.05853866537412008
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,2,512,32,0,0.021162666380405426
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,2,512,64,0,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,2,1024,1,0,0.8234666983286539
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,2,1024,2,0,0.39082666238149005
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,2,1024,8,0,0.09949866930643718
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,2,1024,32,0,0.03669333209594091
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,2,1024,16,0,0.05905066430568695
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,2,1024,64,0,0.034815999368826546
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,2,1024,4,0,0.17561600605646768
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,2,1024,1,0,0.8195412953694662
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,2,1024,2,0,0.3979946772257487
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,2,1024,8,0,0.0993280013402303
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,2,1024,4,0,0.16913066307703653
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,2,1024,16,0,0.058880001306533813
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,2,1024,32,0,0.03669333209594091
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,2,1024,64,0,0.03498666733503342
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,2,1536,1,0,1.6063146591186523
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,2,1536,2,0,0.7633919715881348
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,2,1536,4,0,0.38126933574676514
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,2,1536,8,0,0.1879040002822876
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,2,1536,32,0,0.06758399804433186
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,2,1536,16,0,0.11178666353225708
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,2,1536,64,0,0.049322664737701416
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,2,1536,1,0,1.5738879839579265
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,2,1536,2,0,0.7599786917368571
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,2,1536,4,0,0.3729066848754883
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,2,1536,16,0,0.11212799946467082
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,2,1536,8,0,0.18926932414372763
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,2,1536,32,0,0.06741333504517873
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,2,1536,64,0,0.04949333270390829
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,2,2048,2,0,1.2649813493092854
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,2,2048,1,0,2.581845283508301
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,2,2048,8,0,0.304639995098114
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,2,2048,4,0,0.6311253309249878
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,2,2048,16,0,0.176639993985494
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,2,2048,32,0,0.116565336783727
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,2,2048,64,0,0.06382933259010315
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,2,2048,2,0,1.2648106416066487
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,2,2048,1,0,2.6699094772338867
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,2,2048,4,0,0.6132053136825562
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,2,2048,8,0,0.3036160071690877
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,2,2048,16,0,0.18056533734003702
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,2,2048,32,0,0.11485866705576579
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,2,2048,64,0,0.06348800162474315
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,2,3072,1,0,5.760512034098308
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,2,3072,4,0,1.32369065284729
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,2,3072,16,0,0.3341653347015381
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,2,3072,2,0,2.6840747197469077
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,2,3072,8,0,0.6568959951400757
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,2,3072,32,0,0.20462934176127115
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,2,3072,64,0,0.13209600249926248
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,2,3072,1,0,5.787477493286133
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,2,3072,2,0,2.6946560541788735
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,2,3072,4,0,1.3199360370635986
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,2,3072,8,0,0.6638933420181274
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,2,3072,16,0,0.3324586749076843
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,2,3072,64,0,0.13363200426101685
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,2,4096,4,0,2.268330732981364
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,2,4096,8,0,1.1316906611124675
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,2,4096,2,0,4.79692808787028
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,2,4096,1,0,10.078549067179361
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,2,3072,32,0,0.20104533433914185
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,2,4096,16,0,0.5604693492253622
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,2,4096,32,0,0.31470932563145954
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,2,4096,64,0,0.20548266172409058
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,2,4096,8,0,1.1146240234375
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,2,4096,32,0,0.31539199749628705
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,2,4096,4,0,2.305023988087972
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,2,4096,16,0,0.5550080140431722
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,2,4096,1,0,9.961130777994791
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,2,4096,64,0,0.20480000972747803
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,2,4096,2,0,4.52403195699056
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,2,6144,4,0,5.071530659993489
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,2,6144,1,0,21.716651916503906
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,2,6144,2,0,10.788010915120443
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,2,6144,16,0,1.1876693566640217
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,2,6144,8,0,2.4468480745951333
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,2,6144,32,0,0.621397336324056
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,2,6144,64,0,0.36420265833536786
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,2,6144,8,0,2.369706630706787
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,2,6144,4,0,5.280255953470866
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,2,6144,16,0,1.1912533442179363
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,2,6144,64,0,0.3662506739298503
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,2,6144,32,0,0.6275413433710734
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,2,6144,1,0,22.060373942057293
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,2,6144,2,0,10.533546447753906
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,2,8192,32,0,1.1217919985453289
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,2,8192,16,0,2.1237759590148926
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,2,8192,8,0,4.197205225626628
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,2,8192,64,0,0.5925546487172445
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,2,8192,4,0,9.102165222167969
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,2,8192,1,0,36.70783996582031
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,2,8192,2,0,18.10466130574544
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,2,8192,16,0,2.10807466506958
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,2,8192,4,0,9.150976181030273
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,2,8192,8,0,4.115285237630208
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,2,8192,32,0,1.1402239799499512
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,2,8192,1,0,36.73770650227865
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,2,8192,64,0,0.5886293252309164
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,2,8192,2,0,18.418346405029297
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,2,10240,16,0,3.2822612126668296
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,2,10240,8,0,6.748671849568685
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,2,10240,32,0,1.6940372784932454
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,2,10240,64,0,0.8622079690297445
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,2,10240,4,0,13.929813385009766
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,2,10240,2,0,28.562090555826824
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,2,10240,8,0,6.539946873982747
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,2,10240,16,0,3.1854934692382812
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,2,10240,32,0,1.6546133359273274
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,2,10240,1,0,57.46892801920573
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,2,10240,4,0,13.933397928873697
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,2,10240,64,0,0.9069226582845052
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,2,10240,2,0,27.990870157877605
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,2,12288,16,0,4.694016138712565
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,2,10240,1,0,57.290751139322914
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,2,12288,8,0,9.956181208292643
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,2,12288,32,0,2.3581013679504395
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,2,12288,64,0,1.3306879997253418
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,2,12288,4,0,19.89410146077474
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,2,12288,2,0,39.34054311116537
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,2,12288,32,0,2.3273812929789224
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,2,12288,16,0,4.674559911092122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,2,12288,8,0,10.022058486938477
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,2,12288,64,0,1.2907520135243733
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,2,12288,4,0,20.085589090983074
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,2,12288,1,0,84.2960205078125
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,2,12288,2,0,39.797078450520836
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,2,16384,16,0,8.694442749023438
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,2,16384,8,0,17.097557067871094
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,2,16384,32,0,4.158975919087728
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,2,16384,64,0,2.1821440060933432
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,2,16384,4,0,35.521705627441406
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,2,12288,1,0,83.24522908528645
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,2,16384,32,0,4.156928062438965
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,2,16384,2,0,72.20736185709636
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,2,16384,8,0,17.401002248128254
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,2,16384,16,0,8.81868807474772
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,2,16384,4,0,34.42176055908203
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,2,16384,64,0,2.06984535853068
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,4,16,2,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,4,16,4,0,0.013264000415802002
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,4,16,1,0,0.024405332903067272
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,4,16,8,0,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,4,16,16,0,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,4,16,32,0,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,2,16384,2,0,71.23080444335938
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,4,16,64,0,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,4,16,1,0,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,4,16,2,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,4,16,4,0,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,4,16,8,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,4,16,32,0,0.00972800018886725
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,4,16,16,0,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,4,16,64,0,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,2,16384,1,0,145.48394775390625
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,4,32,1,0,0.025941332181294758
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,4,32,2,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,4,32,4,0,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,4,32,8,0,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,4,32,16,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,4,32,32,0,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,4,32,64,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,4,32,1,0,0.025941332181294758
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,4,32,2,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,4,32,4,0,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,4,32,8,0,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,4,32,16,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,4,32,32,0,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,4,64,1,0,0.029525332152843475
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,4,32,64,0,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,4,64,2,0,0.018090666582187016
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,2,16384,1,0,143.8018595377604
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,4,64,4,0,0.013621332744757334
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,4,64,8,0,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,4,64,16,0,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,4,64,32,0,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,4,64,64,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,4,64,2,0,0.018090666582187016
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,4,64,1,0,0.029525332152843475
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,4,64,4,0,0.011946666985750198
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,4,64,8,0,0.01722666621208191
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,4,64,16,0,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,4,64,32,0,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,4,64,64,0,0.011402666568756104
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,4,128,1,0,0.053077335158983864
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,4,128,4,0,0.01911466692884763
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,4,128,2,0,0.031744000812371574
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,4,128,8,0,0.012458667159080505
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,4,128,16,0,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,4,128,32,0,0.011434666812419891
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,4,128,1,0,0.052906667192777
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,4,128,64,0,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,4,128,2,0,0.0315733328461647
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,4,128,4,0,0.01911466692884763
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,4,128,8,0,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,4,128,16,0,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,4,128,32,0,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,4,128,64,0,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,4,256,1,0,0.31197865804036456
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,4,256,2,0,0.06911999980608623
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,4,256,4,0,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,4,256,8,0,0.025429333249727886
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,4,256,16,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,4,256,32,0,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,4,256,64,0,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,4,256,1,0,0.3131733338038127
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,4,256,2,0,0.07116800049940745
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,4,256,4,0,0.040106666584809623
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,4,256,8,0,0.025600001215934753
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,4,256,16,0,0.01621333385507266
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,4,256,32,0,0.014677333335081736
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,4,256,64,0,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,4,512,2,0,0.32665600379308063
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,4,512,1,0,0.7246507008870443
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,4,512,4,0,0.10564266641934712
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,4,512,8,0,0.05922133227189382
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,4,512,16,0,0.0365226666132609
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,4,512,32,0,0.023552000522613525
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,4,512,64,0,0.021674667795499165
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,4,512,1,0,0.7241386572519938
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,4,512,2,0,0.3256319959958394
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,4,512,8,0,0.06007466713587443
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,4,512,4,0,0.10598400235176086
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,4,512,16,0,0.0365226666132609
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,4,512,32,0,0.023381332556406658
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,4,512,64,0,0.02184533327817917
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,4,1024,1,0,1.7015466690063477
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,4,1024,2,0,0.8139093716939291
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,4,1024,4,0,0.3930453459421794
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,4,1024,8,0,0.17988266547520956
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,4,1024,16,0,0.10291199882825215
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,4,1024,32,0,0.0602453351020813
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,4,1024,64,0,0.03754666695992152
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,4,1024,1,0,1.702741305033366
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,4,1024,2,0,0.8133973280588785
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,4,1024,4,0,0.39560532569885254
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,4,1024,8,0,0.17100799083709717
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,4,1024,16,0,0.10240000486373901
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,4,1024,32,0,0.06075733403364817
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,4,1024,64,0,0.037717332442601524
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,4,1536,1,0,3.2556374867757163
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,4,1536,2,0,1.6080212593078613
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,4,1536,4,0,0.7579306761423746
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,4,1536,8,0,0.37546666463216144
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,4,1536,16,0,0.18602667252222696
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,4,1536,32,0,0.11485866705576579
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,4,1536,64,0,0.07867733140786488
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,4,1536,1,0,3.2264534632364907
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,4,1536,2,0,1.5872000058492024
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,4,1536,4,0,0.7698773543039957
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,4,1536,8,0,0.37137067317962646
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,4,1536,16,0,0.1884160041809082
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,4,1536,32,0,0.11246933539708455
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,4,1536,64,0,0.07816533247629802
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,4,2048,2,0,2.6146133740743003
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,4,2048,4,0,1.2917760213216145
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,4,2048,1,0,5.558613459269206
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,4,2048,8,0,0.6391466856002808
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,4,2048,16,0,0.3104426662127177
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,4,2048,32,0,0.17834667364756265
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,4,2048,64,0,0.1155413289864858
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,4,2048,1,0,5.625343958536784
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,4,2048,2,0,2.634069283803304
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,4,2048,4,0,1.2827306588490803
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,4,2048,16,0,0.32716800769170123
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,4,2048,8,0,0.6481920083363851
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,4,2048,64,0,0.11690666278203328
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,4,2048,32,0,0.176639993985494
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,4,3072,8,0,1.3428053855895996
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,4,3072,4,0,2.711210568745931
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,4,3072,16,0,0.682154655456543
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,4,3072,2,0,5.783210754394531
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,4,3072,1,0,12.215296427408854
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,4,3072,32,0,0.34167468547821045
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,4,3072,64,0,0.20462934176127115
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,4,3072,1,0,11.923967997233072
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,4,3072,2,0,5.60093879699707
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,4,3072,8,0,1.3566293716430664
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,4,3072,4,0,2.683391888936361
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,4,3072,16,0,0.6905173460642496
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,4,3072,32,0,0.353279987970988
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,4,3072,64,0,0.20753065745035806
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,4,4096,2,0,9.822890599568685
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,4,4096,4,0,4.745557467142741
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,4,4096,16,0,1.1506346861521404
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,4,4096,8,0,2.2872746785481772
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,4,4096,32,0,0.6157653331756592
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,4,4096,1,0,19.853312174479168
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,4,4096,64,0,0.3201706608136495
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,4,4096,8,0,2.25655460357666
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,4,4096,16,0,1.1625813643137615
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,4,4096,4,0,4.766208012898763
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,4,4096,64,0,0.32290132840474445
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,4,4096,2,0,10.106026967366537
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,4,4096,1,0,20.426751454671223
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,4,4096,32,0,0.6178133487701416
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,4,6144,8,0,5.197994550069173
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,4,6144,16,0,2.445312023162842
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,4,6144,32,0,1.2474026679992676
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,4,6144,64,0,0.7307946681976318
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,4,6144,4,0,10.542933146158854
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,4,6144,2,0,21.632171630859375
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,4,6144,1,0,44.464640299479164
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,4,6144,64,0,0.7181653181711832
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,4,6144,8,0,5.012309392293294
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,4,6144,16,0,2.5489066441853843
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,4,6144,32,0,1.2817066510518391
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,4,6144,4,0,10.85149892171224
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,4,6144,2,0,20.84164301554362
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,4,6144,1,0,43.2907969156901
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,4,8192,64,0,1.1600213050842285
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,4,8192,32,0,2.1439146995544434
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,4,8192,16,0,4.3328854242960615
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,4,8192,8,0,9.372330983479818
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,4,8192,4,0,18.332842508951824
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,4,8192,2,0,36.722686767578125
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,4,8192,16,0,4.337151845296224
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,4,8192,32,0,2.1439146995544434
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,4,8192,8,0,8.819199879964193
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,4,8192,1,0,80.70519510904948
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,4,8192,4,0,18.224469502766926
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,4,8192,64,0,1.1892053286234539
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,8,16,1,0,0.04215466479460398
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,8,16,2,0,0.024405332903067272
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,4,8192,2,0,36.78703053792318
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,8,16,4,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,8,16,16,0,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,8,16,8,0,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,8,16,64,0,0.009557333464423815
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,8,16,32,0,0.01108266661564509
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,8,16,1,0,0.04181333382924398
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,4,8192,1,0,79.24292500813802
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,8,16,2,0,0.024234667420387268
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,8,16,4,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,8,16,8,0,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,8,16,16,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,8,16,32,0,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,8,16,64,0,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,8,32,1,0,0.045226668318112694
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,8,32,4,0,0.016384000579516094
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,8,32,2,0,0.0266239990790685
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,8,32,8,0,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,8,32,16,0,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,8,32,64,0,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,8,32,32,0,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,8,32,1,0,0.04505600035190582
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,8,32,2,0,0.02611200014750163
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,8,32,4,0,0.01621333385507266
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,8,32,8,0,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,8,32,16,0,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,8,32,32,0,0.011402666568756104
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,8,64,1,0,0.0506879985332489
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,8,32,64,0,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,8,64,4,0,0.01826133330663045
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,8,64,2,0,0.029525332152843475
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,8,64,8,0,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,8,64,16,0,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,8,64,32,0,0.010410666465759277
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,8,64,64,0,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,8,64,1,0,0.05120000243186951
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,8,64,2,0,0.029696000119050343
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,8,64,8,0,0.011946666985750198
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,8,64,4,0,0.01826133330663045
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,8,64,16,0,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,8,64,32,0,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,8,64,64,0,0.011589333415031433
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,8,128,1,0,0.306005338827769
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,8,128,2,0,0.054272000988324486
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,8,128,4,0,0.032255999743938446
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,8,128,8,0,0.019797333826621372
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,8,128,16,0,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,8,128,32,0,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,8,128,64,0,0.012282667060693106
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,8,128,2,0,0.05358933409055074
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,8,128,4,0,0.031914666295051575
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,8,128,1,0,0.3056640028953552
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,8,128,8,0,0.019797333826621372
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,8,128,16,0,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,8,128,32,0,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,8,128,64,0,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,8,256,1,0,0.7270399729410807
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,8,256,2,0,0.3141973416010539
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,8,256,8,0,0.040448000033696495
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,8,256,4,0,0.07167999943097432
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,8,256,16,0,0.0264533335963885
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,8,256,32,0,0.0170666662355264
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,8,256,64,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,8,256,1,0,0.7256746292114258
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,8,256,2,0,0.31522132953008014
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,8,256,4,0,0.0727040022611618
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,8,256,8,0,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,8,256,32,0,0.016895999511082966
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,8,256,16,0,0.0266239990790685
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,8,256,64,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,8,512,1,0,1.5034027099609375
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,8,512,4,0,0.3362133502960205
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,8,512,2,0,0.7326719760894775
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,8,512,8,0,0.10547199845314026
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,8,512,16,0,0.060415998101234436
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,8,512,32,0,0.03754666695992152
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,8,512,64,0,0.02457600086927414
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,8,512,1,0,1.5032320022583008
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,8,512,2,0,0.7326719760894775
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,8,512,4,0,0.33638401826222736
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,8,512,8,0,0.10717866818110149
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,8,512,16,0,0.06092800199985504
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,8,512,32,0,0.037717332442601524
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,8,512,64,0,0.024405332903067272
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,8,1024,1,0,3.484842618306478
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,8,1024,4,0,0.8314879735310873
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,8,1024,2,0,1.6967679659525554
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,8,1024,8,0,0.40465064843495685
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,8,1024,32,0,0.10274133086204529
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,8,1024,16,0,0.18926932414372763
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,8,1024,64,0,0.062463998794555664
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,8,1024,1,0,3.4751148223876953
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,8,1024,8,0,0.40328534444173175
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,8,1024,2,0,1.7172479629516602
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,8,1024,4,0,0.8311466375986735
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,8,1024,16,0,0.19473065932591757
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,8,1024,32,0,0.10342400272687276
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,8,1024,64,0,0.062463998794555664
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,8,1536,1,0,6.799530665079753
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,8,1536,2,0,3.357525189717611
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,8,1536,4,0,1.597098668416341
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,8,1536,64,0,0.11673600474993388
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,8,1536,32,0,0.1967786749204
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,8,1536,16,0,0.385535995165507
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,8,1536,2,0,3.3022292455037436
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,8,1536,1,0,6.782976150512695
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,8,1536,4,0,1.6517119407653809
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,8,1536,8,0,0.8024746576944987
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,8,1536,8,0,0.7886506716410319
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,8,1536,64,0,0.11451733112335205
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,8,1536,16,0,0.392192006111145
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,8,1536,32,0,0.1960960030555725
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,8,2048,8,0,1.3390506108601887
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,8,2048,4,0,2.73578675587972
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,8,2048,2,0,5.436245600382487
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,8,2048,16,0,0.6621866623560587
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,8,2048,1,0,11.79528554280599
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,8,2048,32,0,0.3454293409983317
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,8,2048,64,0,0.183296004931132
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,8,2048,4,0,2.7327146530151367
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,8,2048,2,0,5.53813362121582
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,8,2048,8,0,1.3153279622395833
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,8,2048,32,0,0.36113067468007404
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,8,2048,16,0,0.6657706499099731
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,8,2048,1,0,11.772416432698568
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,8,2048,64,0,0.1800533334414164
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,8,3072,16,0,1.4161920547485352
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,8,3072,2,0,12.226900736490885
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,8,3072,32,0,0.7145813306172689
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,8,3072,4,0,5.992106755574544
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,8,3072,8,0,2.7253761291503906
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,8,3072,64,0,0.42632532119750977
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,8,3072,1,0,24.11229960123698
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,8,3072,1,0,23.369557698567707
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,8,3072,4,0,5.799936294555664
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,8,3072,16,0,1.4120960235595703
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,8,3072,2,0,12.052480061848959
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,8,3072,64,0,0.4145493507385254
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,8,3072,32,0,0.7188479900360107
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,8,3072,8,0,2.74124813079834
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,8,4096,8,0,4.947797457377116
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,8,4096,4,0,10.442752202351889
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,8,4096,2,0,19.96236801147461
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,8,4096,16,0,2.412714640299479
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,8,4096,64,0,0.673962672551473
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,8,4096,32,0,1.2439893086751301
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,8,4096,1,0,40.344746907552086
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,8,4096,4,0,10.263381322224935
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,8,4096,64,0,0.6840319633483887
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,8,4096,32,0,1.217194636662801
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,16,16,1,0,0.07458133498827617
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,8,4096,16,0,2.388480027516683
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,8,4096,8,0,4.498090744018555
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,8,4096,2,0,20.408490498860676
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,8,4096,1,0,40.39372762044271
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,16,16,2,0,0.04232533276081085
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,16,16,8,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,16,16,4,0,0.024746666351954143
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,16,16,16,0,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,16,16,32,0,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,16,16,2,0,0.04215466479460398
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,16,16,1,0,0.07441066702206929
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,16,16,64,0,0.011061333119869232
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,16,16,4,0,0.024405332903067272
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,16,16,8,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,16,16,32,0,0.010069333637754122
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,16,16,64,0,0.009898666913310686
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,16,32,1,0,0.0795306662718455
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,16,32,2,0,0.045567999283472695
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,16,32,4,0,0.02628266563018163
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,16,32,8,0,0.01621333385507266
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,16,32,16,0,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,16,16,16,0,0.012096000214417776
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,16,32,32,0,0.010581333190202713
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,16,32,1,0,0.07987200220425923
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,16,32,64,0,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,16,32,2,0,0.045567999283472695
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,16,32,4,0,0.02628266563018163
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,16,32,8,0,0.01621333385507266
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,16,32,16,0,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,16,32,64,0,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,16,64,2,0,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,16,64,1,0,0.2954240043958028
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,16,64,8,0,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,16,64,4,0,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,16,64,16,0,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,16,32,32,0,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,16,64,32,0,0.012970666090647379
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,16,64,64,0,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,16,64,1,0,0.29474133253097534
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,16,64,2,0,0.0506879985332489
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,16,64,4,0,0.029866665601730347
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,16,64,8,0,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,16,64,16,0,0.012629333883523941
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,16,64,64,0,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,16,128,1,0,0.731818675994873
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,16,128,2,0,0.31214932600657147
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,16,128,4,0,0.05358933409055074
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,16,64,32,0,0.011264000087976456
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,16,128,8,0,0.032085334261258446
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,16,128,16,0,0.02065066620707512
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,16,128,64,0,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,16,128,32,0,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,16,128,1,0,0.7342080275217692
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,16,128,2,0,0.3136853377024333
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,16,128,4,0,0.05358933409055074
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,16,128,16,0,0.020479999482631683
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,16,128,8,0,0.032085334261258446
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,16,128,32,0,0.013994666437307993
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,16,128,64,0,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,16,256,1,0,1.484458605448405
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,16,256,2,0,0.7314773400624593
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,16,256,4,0,0.3295573393503825
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,16,256,8,0,0.07167999943097432
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,16,256,16,0,0.04130133241415024
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,16,256,32,0,0.027477333943049114
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,16,256,64,0,0.01877333347996076
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,16,256,1,0,1.482752005259196
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,16,256,2,0,0.730282704035441
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,16,256,4,0,0.3283626635869344
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,16,256,8,0,0.07833600044250488
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,16,256,16,0,0.04147200038035711
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,16,256,32,0,0.027306665976842243
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,16,256,64,0,0.018090666582187016
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,16,512,2,0,1.5127894083658855
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,16,512,1,0,3.020458539326986
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,16,512,4,0,0.754858652750651
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,16,512,32,0,0.062463998794555664
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,16,512,16,0,0.12407466769218445
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,16,512,8,0,0.35140268007914227
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,16,512,64,0,0.03942399968703588
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,16,512,4,0,0.7536640167236328
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,16,512,2,0,1.5126187006632488
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,16,512,1,0,3.022848129272461
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,16,512,8,0,0.35123201211293537
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,16,512,32,0,0.062463998794555664
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,16,512,64,0,0.03942399968703588
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,16,512,16,0,0.12168533603350322
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,16,1024,1,0,6.9829972585042315
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,16,1024,2,0,3.4880854288736978
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,16,1024,4,0,1.7278292973836262
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,16,1024,8,0,0.865109364191691
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,16,1024,16,0,0.4360533157984416
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,16,1024,64,0,0.10717866818110149
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,16,1024,32,0,0.2336426575978597
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,16,1024,2,0,3.446784019470215
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,16,1024,1,0,6.933333079020183
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,16,1024,4,0,1.7455786069234211
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,16,1024,32,0,0.23432532946268717
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,16,1024,16,0,0.4416853189468384
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,16,1024,8,0,0.86135466893514
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,16,1024,64,0,0.106495996316274
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,16,1536,4,0,3.312981287638346
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,16,1536,2,0,6.645589192708333
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,16,1536,1,0,14.050987243652344
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,16,1536,8,0,1.7037653923034668
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,16,1536,16,0,0.8574293454488119
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,16,1536,32,0,0.44544001420338947
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,16,1536,64,0,0.2616320053736369
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,16,1536,1,0,13.839701334635416
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,16,1536,2,0,6.735530853271484
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,16,1536,16,0,0.859818696975708
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,16,1536,4,0,3.2960853576660156
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,16,1536,8,0,1.6863573392232258
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,16,1536,32,0,0.4505600134531657
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,16,1536,64,0,0.26231465737024945
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,16,2048,2,0,11.283626556396484
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,16,2048,8,0,2.7973972956339517
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,16,2048,64,0,0.43110398451487225
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,16,2048,32,0,0.7511040369669596
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,16,2048,1,0,22.69269307454427
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,16,2048,16,0,1.3986132939656575
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,16,2048,4,0,5.5780690511067705
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,16,2048,32,0,0.7377920150756836
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,16,2048,4,0,5.469184239705403
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,16,2048,2,0,11.748522440592447
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,16,2048,1,0,23.11475118001302
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,16,2048,8,0,2.7973972956339517
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,16,2048,64,0,0.40994131565093994
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,32,16,1,0,0.1384106675783793
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,16,2048,16,0,1.4161920547485352
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,32,16,2,0,0.07441066702206929
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,32,16,8,0,0.024746666351954143
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,32,16,4,0,0.04249600072701772
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,32,16,64,0,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,32,16,32,0,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,32,16,2,0,0.07441066702206929
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,32,16,1,0,0.13823999961217245
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,32,16,16,0,0.018090666582187016
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,32,16,4,0,0.04232533276081085
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,32,16,8,0,0.02491733431816101
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,32,16,16,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,32,16,32,0,0.010922666639089584
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,32,16,64,0,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,32,32,2,0,0.08004266520341237
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,32,32,4,0,0.045567999283472695
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,32,32,1,0,0.292522668838501
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,32,32,16,0,0.016895999511082966
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,32,32,8,0,0.026965332527955372
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,32,32,32,0,0.011605333536863327
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,32,32,64,0,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,32,32,1,0,0.29286400477091473
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,32,32,2,0,0.07987200220425923
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,32,32,4,0,0.045738667249679565
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,32,32,16,0,0.016895999511082966
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,32,32,8,0,0.026965332527955372
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,32,32,64,0,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,32,64,2,0,0.303274671236674
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,32,64,1,0,0.7435946464538574
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,32,32,32,0,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,32,64,4,0,0.051029334465662636
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,32,64,8,0,0.030207999050617218
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,32,64,16,0,0.01911466692884763
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,32,64,32,0,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,32,64,1,0,0.741546630859375
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,32,64,64,0,0.01331199953953425
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,32,64,2,0,0.3027626673380534
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,32,64,8,0,0.030037333567937214
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,32,64,16,0,0.019285333653291065
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,32,64,4,0,0.051370665431022644
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,32,64,64,0,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,32,64,32,0,0.013141332815090815
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,32,128,1,0,1.4900906880696614
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,32,128,2,0,0.7376213073730469
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,32,128,4,0,0.3252906600634257
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,32,128,8,0,0.05529599885145823
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,32,128,16,0,0.03293866664171219
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,32,128,32,0,0.021503999829292297
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,32,128,64,0,0.016544000556071598
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,32,128,1,0,1.4895787239074707
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,32,128,2,0,0.7335253556569418
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,32,128,4,0,0.32409600416819256
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,32,128,8,0,0.05751466751098633
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,32,128,16,0,0.03310933212439219
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,32,128,32,0,0.027269333600997925
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,32,128,64,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,32,256,4,0,0.7543466885884603
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,32,256,2,0,1.4948693911234539
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,32,256,1,0,2.9617493947347007
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,32,256,8,0,0.3471359809239705
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,32,256,16,0,0.08755200107892354
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,32,256,64,0,0.02918400118748347
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,32,256,32,0,0.043178667624791466
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,32,256,1,0,2.9644800821940103
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,32,256,2,0,1.4931626319885254
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,32,256,8,0,0.34867199261983234
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,32,256,4,0,0.754688024520874
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,32,256,16,0,0.09011200070381165
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,32,256,64,0,0.0290133332212766
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,32,256,32,0,0.043007999658584595
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,32,512,1,0,6.01634152730306
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,32,512,2,0,3.0405972798665366
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,32,512,4,0,1.5402666727701824
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,32,512,16,0,0.3985066811243693
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,32,512,8,0,0.7830186684926351
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,32,512,32,0,0.17800533771514893
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,32,512,64,0,0.06638933221499126
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,32,512,2,0,3.043498675028483
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,32,512,1,0,6.012757619222005
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,32,512,8,0,0.7826773325602213
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,32,512,4,0,1.5438507397969563
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,32,512,16,0,0.3964586655298869
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,32,512,32,0,0.19438934326171875
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,32,512,64,0,0.06587733328342438
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,32,1024,1,0,14.169940948486328
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,32,1024,8,0,1.8213547070821126
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,32,1024,2,0,7.015253067016602
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,32,1024,4,0,3.495253245035807
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,32,1024,32,0,0.5022720098495483
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,32,1024,64,0,0.2983253399531047
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,32,1024,16,0,0.9446400006612142
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,32,1024,1,0,14.4988161722819
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,32,1024,8,0,1.82476806640625
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,32,1024,2,0,7.013205210367839
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,32,1024,4,0,3.5613012313842773
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,32,1024,32,0,0.502613345781962
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,32,1024,16,0,0.9323519865671793
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,32,1024,64,0,0.2988373239835103
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,64,16,1,0,0.33297065893809
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,64,16,2,0,0.1687893271446228
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,64,16,4,0,0.07526400188604991
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,64,16,8,0,0.04334933559099833
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,64,16,16,0,0.025600001215934753
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,64,16,32,0,0.01621333385507266
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,64,16,1,0,0.3346773386001587
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,64,16,64,0,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,64,16,2,0,0.13994666934013367
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,64,16,8,0,0.04266666869322459
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,64,16,4,0,0.07526400188604991
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,64,16,16,0,0.025087999800841015
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,64,16,32,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,64,16,64,0,0.01109333336353302
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,64,32,1,0,0.7174826463063558
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,64,32,4,0,0.08106666803359985
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,64,32,2,0,0.29764266808827716
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,64,32,8,0,0.04625066618124644
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,64,32,16,0,0.027647999425729115
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,64,32,64,0,0.01228800043463707
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,64,32,32,0,0.01757866640885671
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,64,32,1,0,0.71833602587382
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,64,32,2,0,0.3199999928474426
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,64,32,4,0,0.08106666803359985
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,64,32,16,0,0.027477333943049114
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,64,32,8,0,0.04642133414745331
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,64,32,32,0,0.017407999684413273
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,64,32,64,0,0.012117333710193634
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,64,64,1,0,1.5047680536905925
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,64,64,2,0,0.7476906776428223
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,64,64,4,0,0.318122665087382
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,64,64,8,0,0.053930665055910744
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,64,64,32,0,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,64,64,16,0,0.0314026673634847
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,64,64,64,0,0.0145066666106383
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,64,64,1,0,1.5037439664204915
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,64,64,2,0,0.749397357304891
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,64,64,8,0,0.05358933409055074
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,64,64,4,0,0.31726932525634766
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,64,64,16,0,0.031744000812371574
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,64,64,64,0,0.014335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,64,128,1,0,2.973525365193685
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,64,64,32,0,0.020309332758188248
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,64,128,4,0,0.7521279652913412
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,64,128,2,0,1.5042559305826824
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,64,128,8,0,0.3490133285522461
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,64,128,16,0,0.08499200145403545
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,64,128,64,0,0.023893333971500397
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,64,128,32,0,0.03515733281771342
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,64,128,1,0,2.975402514139811
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,64,128,8,0,0.3490133285522461
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,64,128,16,0,0.07799466451009114
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,64,128,4,0,0.756223996480306
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,64,128,32,0,0.03532800078392029
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,64,128,64,0,0.02372266600529353
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,64,128,2,0,1.5047680536905925
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,64,256,2,0,2.9823999404907227
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,64,256,4,0,1.527125358581543
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,64,256,1,0,5.892778396606445
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,64,256,8,0,0.7838719685872396
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,64,256,32,0,0.17254400253295898
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,64,256,16,0,0.4007253249486287
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,64,256,64,0,0.04727466901143392
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,64,256,4,0,1.5276373227437336
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,64,256,16,0,0.398848017056783
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,64,256,2,0,2.9844481150309243
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,64,256,32,0,0.1718613306681315
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,64,256,8,0,0.78438401222229
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,64,256,64,0,0.04727466901143392
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,64,256,1,0,5.889706929524739
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,64,512,1,0,12.017152150472006
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,64,512,8,0,1.6054612795511882
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,64,512,2,0,6.075562795003255
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,64,512,16,0,0.8656213283538818
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,64,512,4,0,3.0882132848103843
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,64,512,64,0,0.283135990301768
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,64,512,32,0,0.4896426598230998
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,64,512,2,0,6.072320302327474
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,64,512,8,0,1.6104106903076172
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,64,512,16,0,0.8625493049621582
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,64,512,4,0,3.0778026580810547
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,64,512,32,0,0.5179733435312907
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,64,512,64,0,0.2845013340314229
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,128,16,1,0,0.7179946899414062
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,64,512,1,0,12.028074900309244
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,128,16,2,0,0.34815998872121173
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,128,16,4,0,0.14335999886194864
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,128,16,8,0,0.07628799974918365
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,128,16,64,0,0.016895999511082966
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,128,16,32,0,0.0264533335963885
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,128,16,16,0,0.04420266548792521
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,128,16,1,0,0.7208960056304932
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,128,16,4,0,0.14131200313568115
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,128,16,8,0,0.07628799974918365
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,128,16,16,0,0.04369066655635834
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,128,16,32,0,0.02611200014750163
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,128,16,64,0,0.01672533278663953
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,128,16,2,0,0.34884266058603924
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,128,32,1,0,1.4641493161519368
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,128,32,4,0,0.3109546701113383
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,128,32,8,0,0.08430932958920796
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,128,32,2,0,0.7268693447113037
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,128,32,16,0,0.04778666794300079
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,128,32,32,0,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,128,32,64,0,0.018602666755517323
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,128,32,4,0,0.3118079900741577
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,128,32,2,0,0.7316479682922363
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,128,32,8,0,0.08379733562469482
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,128,32,16,0,0.048298666874567665
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,128,32,32,0,0.0288426677385966
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,128,32,64,0,0.018944000204404194
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,128,32,1,0,1.4685866038004558
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,128,64,1,0,2.9911041259765625
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,128,64,2,0,1.5133013725280762
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,128,64,8,0,0.3447466691335042
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,128,64,4,0,0.7705600261688232
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,128,64,16,0,0.08004266520341237
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,128,64,32,0,0.03379199902216593
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,128,64,64,0,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,128,64,2,0,1.519445260365804
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,128,64,4,0,0.7714133262634277
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,128,64,8,0,0.3423573176066081
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,128,64,1,0,2.984959920247396
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,128,64,16,0,0.07765333354473114
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,128,64,32,0,0.0339626669883728
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,128,64,64,0,0.02252800017595291
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,128,128,1,0,5.905749638875325
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,128,128,8,0,0.7946240107218424
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,128,128,4,0,1.5339520772298176
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,128,128,2,0,2.9986133575439453
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,128,128,32,0,0.17390932639439902
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,128,128,64,0,0.04027733455101649
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,128,128,16,0,0.40192000071207684
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,128,128,1,0,5.900288263956706
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,128,128,2,0,2.99946657816569
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,128,128,8,0,0.7936000029246012
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,128,128,4,0,1.5346345901489258
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,128,128,16,0,0.40345601240793866
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,128,128,32,0,0.1730560064315796
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,128,128,64,0,0.03976533313592275
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,128,256,2,0,5.936639785766602
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,128,256,4,0,3.046229362487793
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,128,256,8,0,1.5952213605244954
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,128,256,16,0,0.8688639799753824
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,128,256,1,0,11.769002278645834
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,128,256,32,0,0.4962986707687378
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,128,256,64,0,0.28654932975769043
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,128,256,1,0,11.772586822509766
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,128,256,2,0,5.940053304036458
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,128,256,16,0,0.8673280080159506
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,128,256,8,0,1.598464012145996
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,128,256,64,0,0.2892799973487854
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,128,256,32,0,0.520362655321757
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,128,256,4,0,3.0498132705688477
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,256,16,1,0,1.5378774007161458
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,256,16,2,0,0.7581013043721517
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,256,16,4,0,0.35942399501800537
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,256,16,8,0,0.14813866217931113
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,256,16,16,0,0.07799466451009114
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,256,16,64,0,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,256,16,2,0,0.740010658899943
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,256,16,32,0,0.04539733131726583
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,256,16,1,0,1.5371947288513184
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,256,16,8,0,0.1460906664530436
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,256,16,4,0,0.34884266058603924
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,256,16,32,0,0.04539733131726583
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,256,16,16,0,0.07833600044250488
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,256,16,64,0,0.027989332874615986
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,256,32,8,0,0.33740798632303876
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,256,32,2,0,1.489408016204834
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,256,32,16,0,0.09898666540781657
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,256,32,4,0,0.7497386932373047
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,256,32,1,0,2.9289814631144204
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,256,32,32,0,0.05017599960168203
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,256,32,64,0,0.031231999397277832
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,256,32,1,0,2.926250775655111
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,256,32,2,0,1.4854826927185059
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,256,32,8,0,0.33604268232981366
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,256,32,4,0,0.7492266496022543
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,256,32,16,0,0.09830400347709656
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,256,32,32,0,0.0506879985332489
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,256,32,64,0,0.031231999397277832
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,256,64,2,0,3.0156799952189126
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,256,64,4,0,1.5540906588236492
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,256,64,1,0,5.897898356119792
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,256,64,8,0,0.8075946966807047
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,256,64,32,0,0.17407999436060587
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,256,64,16,0,0.40482131640116376
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,256,64,64,0,0.03839999934037527
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,256,64,1,0,5.937493642171224
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,256,64,2,0,3.021482785542806
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,256,64,4,0,1.5542613665262859
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,256,64,8,0,0.8297812938690186
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,256,64,32,0,0.17254400253295898
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,256,64,16,0,0.40277334054311115
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,256,64,64,0,0.03788800040880839
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,128,256,128,1,0,11.787946065266928
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,16,256,128,8,0,1.607338587443034
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,64,256,128,2,0,5.97316296895345
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,32,256,128,4,0,3.0632960001627603
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,4,256,128,32,0,0.504149317741394
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,2,256,128,64,0,0.2926933368047078
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,fp8,8,256,128,16,0,0.8772266705830892
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,128,256,128,1,0,11.789653778076172
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,16,256,128,8,0,1.6855039596557617
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,64,256,128,2,0,5.972991943359375
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,8,256,128,16,0,0.9106773535410563
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,32,256,128,4,0,3.060394605000814
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,4,256,128,32,0,0.5065386692682902
TRTLLM,1.0.0,NVIDIA L40S,mla_context,default,float16,float16,2,256,128,64,0,0.292522668838501
