framework,version,device,op_name,kernel_source,bmm_dtype,num_tokens,num_heads,latency
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,1,128,0.010547199845314026
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,1,128,0.007372800260782242
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,1,64,0.008499199897050858
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,1,64,0.007952000200748443
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,1,32,0.006140799820423126
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,1,32,0.00982080027461052
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,1,8,0.005324799939990044
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,1,16,0.006041600182652473
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,1,16,0.005734400078654289
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,1,8,0.005734400078654289
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,1,1,0.005084799975156784
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,1,4,0.005836800113320351
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,1,1,0.005836800113320351
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,2,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,1,4,0.006041600182652473
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,1,2,0.006639999896287918
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,2,128,0.009216000139713288
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,2,64,0.006143999844789505
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,1,2,0.006758400052785873
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,2,64,0.00727040022611618
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,2,32,0.005820799991488457
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,2,32,0.005836800113320351
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,2,4,0.005632000043988228
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,2,8,0.00570559985935688
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,2,2,0.005737600103020668
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,2,16,0.005715199932456017
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,2,8,0.006246399879455566
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,2,16,0.005734400078654289
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,2,4,0.006041600182652473
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,2,2,0.006451199948787689
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,2,1,0.006041600182652473
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,2,1,0.0051968000829219815
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,4,128,0.009318400174379349
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,4,64,0.006041600182652473
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,4,32,0.005939200147986412
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,4,64,0.00727040022611618
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,4,128,0.010329599678516387
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,4,32,0.005827200040221215
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,4,16,0.005939200147986412
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,4,16,0.005529600009322166
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,4,8,0.006451199948787689
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,4,4,0.005836800113320351
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,4,8,0.00634239986538887
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,4,4,0.00551999993622303
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,4,1,0.006041600182652473
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,4,2,0.005910399928689003
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,4,1,0.005324799939990044
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,8,128,0.009318400174379349
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,4,2,0.005632000043988228
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,8,128,0.010444799810647965
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,8,64,0.006143999844789505
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,8,64,0.00727040022611618
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,8,32,0.005836800113320351
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,8,16,0.005119999870657921
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,8,16,0.005939200147986412
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,8,32,0.005836800113320351
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,8,8,0.005836800113320351
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,8,8,0.006348799914121628
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,8,4,0.005756799876689911
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,8,4,0.005939200147986412
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,8,2,0.005939200147986412
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,8,2,0.005222399905323982
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,8,1,0.008806400001049042
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,16,128,0.010649599879980088
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,8,1,0.005939200147986412
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,16,128,0.009743999689817429
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,16,32,0.005939200147986412
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,16,64,0.006348799914121628
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,16,64,0.007577600330114365
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,16,32,0.005932800099253654
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,16,16,0.006553599983453751
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,16,16,0.005939200147986412
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,16,8,0.005939200147986412
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,16,8,0.005632000043988228
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,16,4,0.005836800113320351
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,16,4,0.005836800113320351
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,16,2,0.005404800176620483
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,16,1,0.006348799914121628
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,16,2,0.006553599983453751
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,16,1,0.005836800113320351
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,32,128,0.010444799810647965
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,32,128,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,32,64,0.006857600063085556
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,32,32,0.005203200131654739
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,32,64,0.007782399654388428
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,32,32,0.006041600182652473
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,32,16,0.005939200147986412
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,32,16,0.008396799862384795
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,32,8,0.005939200147986412
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,32,8,0.005939200147986412
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,32,4,0.006041600182652473
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,32,4,0.005529600009322166
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,32,2,0.005836800113320351
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,32,2,0.005849599838256836
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,32,1,0.005836800113320351
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,32,1,0.006041600182652473
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,48,64,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,48,128,0.018432000279426576
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,48,128,0.017612800002098083
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,48,64,0.010940799862146378
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,48,32,0.006656000018119812
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,48,16,0.007680000364780426
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,48,32,0.007782399654388428
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,48,16,0.006041600182652473
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,48,8,0.0064640000462532045
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,48,8,0.005734400078654289
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,48,4,0.006041600182652473
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,48,4,0.005529600009322166
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,48,2,0.006451199948787689
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,48,2,0.005836800113320351
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,48,1,0.005737600103020668
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,64,128,0.019046400487422944
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,48,1,0.005833600088953972
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,64,128,0.018432000279426576
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,64,32,0.006758400052785873
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,64,64,0.011673600226640702
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,64,64,0.010444799810647965
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,64,32,0.007983999699354172
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,64,16,0.0058143999427556995
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,64,16,0.006444799900054932
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,64,8,0.006361600011587143
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,64,8,0.0061471998691558834
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,64,4,0.005836800113320351
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,64,4,0.0055263999849557875
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,64,2,0.005939200147986412
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,64,2,0.006041600182652473
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,64,1,0.005744000151753426
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,64,1,0.005939200147986412
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,80,128,0.025190401077270507
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,80,64,0.014847999811172486
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,80,128,0.02385919988155365
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,80,64,0.014617599546909332
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,80,32,0.009318400174379349
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,80,32,0.008588799834251403
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,80,16,0.005836800113320351
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,80,16,0.007475200295448303
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,80,8,0.005836800113320351
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,80,4,0.005734400078654289
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,80,8,0.005827200040221215
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,80,4,0.005632000043988228
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,80,2,0.005939200147986412
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,80,2,0.00594559982419014
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,80,1,0.006348799914121628
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,80,1,0.005846399813890457
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,96,128,0.025804799795150758
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,96,128,0.02457599937915802
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,96,32,0.008703999966382981
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,96,64,0.014847999811172486
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,96,64,0.01525759994983673
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,96,32,0.0095040000975132
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,96,16,0.005939200147986412
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,96,16,0.007475200295448303
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,96,8,0.005836800113320351
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,96,8,0.005939200147986412
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,96,4,0.0058432001620531086
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,96,4,0.005734400078654289
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,96,2,0.0059424001723527905
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,96,1,0.00584000013768673
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,96,2,0.005638400092720985
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,96,1,0.005811199918389321
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,128,128,0.03318400084972382
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,128,64,0.018943999707698823
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,128,128,0.030822399258613586
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,128,64,0.018432000279426576
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,128,32,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,128,32,0.010444799810647965
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,128,16,0.006656000018119812
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,128,16,0.007884799689054488
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,128,8,0.005939200147986412
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,128,8,0.006041600182652473
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,128,4,0.0057472001761198046
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,128,4,0.005734400078654289
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,128,2,0.006553599983453751
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,128,2,0.005836800113320351
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,128,1,0.005638400092720985
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,128,1,0.005836800113320351
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,160,128,0.04095999896526337
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,160,128,0.037887999415397645
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,160,64,0.023039999604225158
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,160,64,0.021484799683094025
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,160,32,0.013414399325847625
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,160,32,0.013395200669765472
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,160,16,0.009187199920415879
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,160,16,0.007680000364780426
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,160,8,0.005836800113320351
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,160,8,0.007372800260782242
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,160,2,0.005939200147986412
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,160,4,0.005936000123620033
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,160,4,0.005836800113320351
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,160,2,0.006143999844789505
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,160,1,0.005721599981188774
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,160,1,0.005836800113320351
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,192,128,0.03061760067939758
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,192,128,0.04392960071563721
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,192,64,0.017715199291706084
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,192,32,0.010447999835014344
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,192,64,0.02457599937915802
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,192,32,0.014950400590896607
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,192,16,0.008703999966382981
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,192,8,0.007475200295448303
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,192,16,0.009523200243711472
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,192,8,0.006963200122117996
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,192,4,0.006041600182652473
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,192,4,0.005836800113320351
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,192,2,0.005734400078654289
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,192,2,0.005939200147986412
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,192,1,0.005939200147986412
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,192,1,0.005734400078654289
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,256,128,0.05734400153160095
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,256,128,0.03266560137271881
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,256,64,0.01884160041809082
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,256,64,0.030720001459121703
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,256,32,0.010649599879980088
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,256,32,0.018432000279426576
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,256,16,0.026419198513031004
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,256,16,0.01053439974784851
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,256,8,0.006758400052785873
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,256,8,0.007884799689054488
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,256,4,0.006246399879455566
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,256,4,0.006041600182652473
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,256,2,0.005734400078654289
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,256,2,0.005939200147986412
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,256,1,0.005939200147986412
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,256,1,0.005939200147986412
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,320,128,0.04259839951992035
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,320,64,0.024371199309825897
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,320,64,0.03768320083618164
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,320,128,0.07086079716682434
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,320,32,0.01454080045223236
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,320,16,0.010342399775981902
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,320,8,0.007568000257015229
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,320,16,0.013312000036239623
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,320,32,0.02170239984989166
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,320,8,0.009113600105047226
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,320,4,0.005119999870657921
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,320,4,0.007372800260782242
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,320,2,0.0070880003273487095
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,320,2,0.005836800113320351
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,320,1,0.005939200147986412
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,320,1,0.005836800113320351
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,384,128,0.04567039906978607
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,384,128,0.08417279720306396
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,384,64,0.026009601354599
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,384,64,0.04392960071563721
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,384,32,0.015052799880504609
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,384,32,0.024371199309825897
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,384,16,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,384,16,0.014950400590896607
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,384,8,0.008601599931716919
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,384,8,0.009523200243711472
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,384,4,0.005632000043988228
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,384,4,0.007475200295448303
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,384,2,0.005734400078654289
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,384,2,0.005939200147986412
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,384,1,0.005836800113320351
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,384,1,0.007065600156784058
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,512,128,0.11151360273361206
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,512,128,0.07946239709854126
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,512,64,0.03266560137271881
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,512,32,0.018739199638366698
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,512,64,0.05744640231132507
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,512,32,0.030924800038337707
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,512,16,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,512,16,0.018432000279426576
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,512,8,0.010547199845314026
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,512,8,0.011776000261306763
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,512,4,0.006656000018119812
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,512,4,0.007680000364780426
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,512,2,0.005939200147986412
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,512,2,0.006041600182652473
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,512,1,0.005939200147986412
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,768,128,0.23746559619903565
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,512,1,0.005711999908089638
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,768,64,0.04669440090656281
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,768,128,0.1610751986503601
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,768,64,0.08304640054702758
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,768,32,0.04341759979724884
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,768,16,0.014847999811172486
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,768,32,0.025600001215934753
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,768,16,0.02447360008955002
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,768,8,0.010239999741315842
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,768,8,0.01462399959564209
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,768,4,0.008601599931716919
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,768,4,0.009318400174379349
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,768,2,0.005836800113320351
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,768,2,0.007372800260782242
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,768,1,0.005836800113320351
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,768,1,0.006252799928188324
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,1024,128,0.21155838966369628
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,1024,128,0.33402879238128663
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,1024,64,0.06789119839668274
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,1024,64,0.1087488055229187
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,1024,32,0.03287039995193482
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,1024,32,0.05611519813537598
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,1024,16,0.03020800054073334
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,1024,16,0.018739199638366698
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,1024,8,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,1024,8,0.01802240014076233
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,1024,4,0.010444799810647965
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,1024,4,0.011555200070142746
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,1024,2,0.006748799979686737
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,1024,2,0.007782399654388428
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,1024,1,0.005939200147986412
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,1024,1,0.006041600182652473
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,1536,128,0.5101568222045898
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,1536,128,0.31139841079711916
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,1536,64,0.15738879442214965
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,1536,64,0.21350400447845458
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,1536,32,0.045158401131629944
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,1536,32,0.08120319843292237
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,1536,16,0.02539519965648651
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,1536,8,0.014950400590896607
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,1536,16,0.04259839951992035
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,1536,8,0.024035200476646423
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,1536,4,0.01022080034017563
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,1536,2,0.008499199897050858
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,1536,4,0.01451839953660965
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,1536,2,0.00941760018467903
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,1536,1,0.006246399879455566
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,1536,1,0.007372800260782242
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,2048,128,0.46346240043640136
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,2048,128,0.6819839954376221
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,2048,64,0.2091007947921753
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,2048,64,0.3156991958618164
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,2048,32,0.10751999616622925
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,2048,16,0.03256320059299469
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,2048,32,0.06748800277709961
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,2048,16,0.05529599785804749
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,2048,8,0.01884160041809082
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,2048,8,0.029798400402069092
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,2048,4,0.010547199845314026
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,2048,4,0.017817600071430205
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,2048,2,0.010137599706649781
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,2048,2,0.011673600226640702
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,2048,1,0.005939200147986412
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,2048,1,0.007756800204515457
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,3072,128,1.022054386138916
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,3072,128,0.7468031883239746
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,3072,64,0.5161983966827393
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,3072,64,0.30853118896484377
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,3072,32,0.20746240615844727
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,3072,32,0.15575040578842164
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,3072,16,0.08038399815559387
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,3072,16,0.045158401131629944
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,3072,8,0.025600001215934753
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,3072,8,0.0421887993812561
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,3072,4,0.014847999811172486
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,3072,4,0.023756800591945647
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,3072,2,0.014336000382900237
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,3072,2,0.010236799716949463
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,3072,1,0.006348799914121628
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,3072,1,0.00942080020904541
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,4096,128,1.364684772491455
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,4096,128,0.9981951713562012
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,4096,64,0.6821887969970704
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,4096,64,0.45168638229370117
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,4096,32,0.29859199523925783
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,4096,32,0.20643839836120606
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,4096,16,0.06796799898147583
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,4096,16,0.10629119873046874
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,4096,8,0.03256320059299469
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,4096,4,0.019046400487422944
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,4096,8,0.05478399991989136
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,4096,4,0.029491201043128967
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,4096,2,0.010649599879980088
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,4096,2,0.017609600722789765
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,4096,1,0.00727040022611618
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,4096,1,0.011468800157308579
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,6144,128,2.028851127624512
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,6144,128,1.492582416534424
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,6144,64,0.9993215560913086
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,6144,64,0.7281663894653321
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,6144,32,0.30074880123138426
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,6144,32,0.506060791015625
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,6144,16,0.1960960030555725
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,6144,16,0.15196160078048707
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,6144,8,0.04976640045642853
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,6144,8,0.07874559760093688
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,6144,4,0.02529279887676239
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,6144,4,0.04095999896526337
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,6144,2,0.016998399794101716
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,6144,2,0.02314240038394928
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,6144,1,0.009932799637317658
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,6144,1,0.014131200313568116
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,8192,128,2.003455924987793
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,8192,128,2.7009023666381835
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,8192,64,0.9772031784057618
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,8192,32,0.6709248065948487
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,8192,64,1.3408255577087402
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,8192,32,0.44328961372375486
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,8192,16,0.29419519901275637
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,8192,16,0.19875839948654175
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,8192,8,0.06512640118598938
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,8192,8,0.12052479982376099
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,8192,4,0.03307519853115082
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,8192,4,0.05273600220680237
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,8192,2,0.018739199638366698
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,8192,2,0.028467199206352232
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,float16,8192,1,0.010649599879980088
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_pre,default,fp8,8192,1,0.016998399794101716
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,1,128,0.008499199897050858
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,1,128,0.006963200122117996
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,1,64,0.006041600182652473
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,1,64,0.006313599646091461
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,1,32,0.005836800113320351
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,1,32,0.006239999830722809
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,1,16,0.006143999844789505
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,1,8,0.010342399775981902
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,1,16,0.007065600156784058
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,1,8,0.006041600182652473
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,1,4,0.008806400001049042
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,1,4,0.007987199723720551
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,1,1,0.005222399905323982
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,1,1,0.006041600182652473
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,2,128,0.008703999966382981
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,1,2,0.00570559985935688
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,1,2,0.005836800113320351
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,2,32,0.005836800113320351
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,2,32,0.006348799914121628
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,2,128,0.006860800087451935
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,2,16,0.005619199946522713
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,2,64,0.006143999844789505
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,2,64,0.0066431999206542965
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,2,16,0.006041600182652473
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,2,8,0.007987199723720551
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,2,4,0.005836800113320351
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,2,8,0.006143999844789505
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,2,2,0.005632000043988228
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,2,1,0.007241600006818771
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,2,4,0.006041600182652473
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,2,2,0.005734400078654289
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,2,1,0.005728000029921532
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,4,128,0.008806400001049042
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,4,128,0.007065600156784058
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,4,64,0.006656000018119812
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,4,64,0.006348799914121628
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,4,32,0.005836800113320351
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,4,32,0.006143999844789505
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,4,16,0.006041600182652473
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,4,8,0.007577600330114365
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,4,8,0.006143999844789505
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,4,4,0.006348799914121628
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,4,4,0.006041600182652473
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,4,16,0.005734400078654289
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,4,2,0.005907199904322624
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,4,1,0.005644800141453743
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,4,2,0.0060479998588562015
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,4,1,0.006441599875688553
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,8,128,0.00891520008444786
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,8,128,0.007782399654388428
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,8,64,0.006758400052785873
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,8,32,0.0060384001582860945
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,8,64,0.006348799914121628
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,8,32,0.0062304001301527025
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,8,16,0.006143999844789505
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,8,8,0.006246399879455566
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,8,8,0.006041600182652473
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,8,4,0.005632000043988228
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,8,16,0.005632000043988228
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,8,2,0.005632000043988228
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,8,4,0.0062431998550891874
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,8,2,0.006041600182652473
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,8,1,0.005836800113320351
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,8,1,0.006143999844789505
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,16,128,0.009216000139713288
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,16,128,0.008499199897050858
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,16,32,0.0060384001582860945
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,16,64,0.006963200122117996
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,16,64,0.00727040022611618
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,16,32,0.006348799914121628
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,16,16,0.005939200147986412
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,16,16,0.006348799914121628
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,16,8,0.005836800113320351
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,16,8,0.006143999844789505
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,16,4,0.005948799848556519
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,16,4,0.006041600182652473
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,16,2,0.005632000043988228
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,16,1,0.005939200147986412
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,16,2,0.006041600182652473
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,16,1,0.006143999844789505
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,32,128,0.010649599879980088
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,32,128,0.009836799651384353
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,32,64,0.007782399654388428
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,32,64,0.008169600367546081
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,32,32,0.006246399879455566
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,32,32,0.007372800260782242
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,32,16,0.006553599983453751
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,32,16,0.005948799848556519
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,32,8,0.006041600182652473
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,32,8,0.006348799914121628
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,32,4,0.0058559998869895935
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,32,1,0.005734400078654289
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,32,4,0.006143999844789505
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,32,2,0.006246399879455566
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,32,2,0.005734400078654289
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,32,1,0.006239999830722809
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,48,128,0.015974399447441102
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,48,128,0.014950400590896607
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,48,64,0.009625600278377533
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,48,64,0.009625600278377533
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,48,32,0.007577600330114365
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,48,16,0.006553599983453751
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,48,16,0.006143999844789505
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,48,32,0.007475200295448303
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,48,8,0.005948799848556519
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,48,8,0.006348799914121628
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,48,4,0.006128000095486641
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,48,4,0.006246399879455566
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,48,2,0.005939200147986412
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,48,2,0.0062304001301527025
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,64,128,0.016998399794101716
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,48,1,0.006143999844789505
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,48,1,0.005119999870657921
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,64,128,0.01730560064315796
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,64,64,0.009932799637317658
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,64,64,0.010751999914646149
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,64,32,0.007782399654388428
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,64,32,0.007987199723720551
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,64,16,0.006143999844789505
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,64,8,0.006451199948787689
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,64,8,0.006041600182652473
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,64,16,0.007168000191450119
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,64,4,0.0059519998729228975
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,64,4,0.006444799900054932
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,64,2,0.005939200147986412
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,64,1,0.005119999870657921
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,64,2,0.006246399879455566
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,64,1,0.006246399879455566
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,80,128,0.02178560048341751
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,80,128,0.022015999257564544
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,80,64,0.013209599256515502
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,80,64,0.014233599603176116
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,80,32,0.008294399827718735
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,80,32,0.008697599917650223
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,80,16,0.00727040022611618
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,80,16,0.007475200295448303
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,80,8,0.006041600182652473
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,80,4,0.006124800071120262
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,80,8,0.006451199948787689
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,80,4,0.006348799914121628
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,80,2,0.005939200147986412
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,80,2,0.006246399879455566
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,80,1,0.005939200147986412
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,80,1,0.006246399879455566
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,96,128,0.024063999950885772
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,96,128,0.022835199534893037
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,96,64,0.013209599256515502
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,96,64,0.01443839967250824
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,96,32,0.008601599931716919
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,96,32,0.009113600105047226
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,96,16,0.007398399710655213
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,96,16,0.007475200295448303
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,96,8,0.006246399879455566
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,96,8,0.006451199948787689
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,96,4,0.005932800099253654
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,96,4,0.006656000018119812
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,96,2,0.006329599767923355
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,96,2,0.006143999844789505
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,96,1,0.006348799914121628
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,96,1,0.005222399905323982
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,128,128,0.02805759906768799
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,128,128,0.03051519989967346
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,128,64,0.017407999932765962
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,128,64,0.01669439971446991
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,128,32,0.009728000313043595
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,128,32,0.010649599879980088
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,128,16,0.007680000364780426
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,128,16,0.007987199723720551
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,128,8,0.006348799914121628
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,128,8,0.00716480016708374
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,128,4,0.006041600182652473
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,128,4,0.006451199948787689
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,128,2,0.005939200147986412
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,128,2,0.006451199948787689
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,128,1,0.00584000013768673
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,128,1,0.006246399879455566
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,160,128,0.03676159977912903
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,160,64,0.019251200556755065
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,160,128,0.03635199964046478
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,160,32,0.011366400122642516
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,160,64,0.021193599700927733
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,160,32,0.013823999464511872
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,160,16,0.008294399827718735
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,160,16,0.008601599931716919
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,160,8,0.00727040022611618
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,160,8,0.007475200295448303
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,160,4,0.006143999844789505
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,160,4,0.006451199948787689
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,160,2,0.006451199948787689
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,160,2,0.00594559982419014
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,160,1,0.005939200147986412
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,160,1,0.0062304001301527025
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,192,128,0.043007999658584595
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,192,128,0.04157440066337585
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,192,64,0.023654399812221526
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,192,64,0.022835199534893037
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,192,32,0.013107199966907502
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,192,32,0.01443839967250824
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,192,16,0.008601599931716919
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,192,16,0.00910400003194809
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,192,8,0.007478400319814682
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,192,8,0.007577600330114365
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,192,4,0.006121600046753883
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,192,4,0.006553599983453751
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,192,2,0.008191999793052674
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,192,2,0.006348799914121628
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,192,1,0.006451199948787689
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,192,1,0.006246399879455566
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,256,128,0.05847039818763733
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,256,128,0.054067200422286986
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,256,64,0.027852800488471986
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,256,32,0.01679359972476959
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,256,64,0.03031040132045746
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,256,16,0.009830400347709656
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,256,32,0.017100800573825837
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,256,16,0.010547199845314026
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,256,8,0.007756800204515457
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,256,8,0.007987199723720551
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,256,4,0.006143999844789505
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,256,4,0.00727040022611618
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,256,2,0.006041600182652473
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,256,2,0.006348799914121628
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,256,1,0.005222399905323982
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,320,128,0.07178239822387696
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,256,1,0.006224000081419945
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,320,128,0.066048002243042
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,320,64,0.03665919899940491
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,320,64,0.03665919899940491
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,320,32,0.01945600062608719
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,320,32,0.021196800470352172
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,320,16,0.013823999464511872
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,320,16,0.011161600053310395
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,320,8,0.008287999778985977
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,320,8,0.008806400001049042
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,320,4,0.0072512000799179075
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,320,4,0.007475200295448303
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,320,2,0.006041600182652473
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,320,2,0.006553599983453751
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,320,1,0.006121600046753883
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,320,1,0.006348799914121628
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,384,128,0.08520320057868958
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,384,128,0.07772160172462464
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,384,64,0.043110400438308716
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,384,64,0.04136959910392761
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,384,32,0.02293439954519272
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,384,32,0.023654399812221526
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,384,16,0.013312000036239623
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,384,16,0.01454080045223236
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,384,8,0.008499199897050858
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,384,8,0.009216000139713288
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,384,4,0.007372800260782242
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,384,4,0.008291199803352356
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,384,2,0.006060799956321717
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,384,2,0.006553599983453751
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,384,1,0.005939200147986412
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,512,128,0.048332801461219786
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,512,128,0.14991359710693358
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,384,1,0.006246399879455566
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,512,64,0.024268800020217897
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,512,64,0.053964799642562865
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,512,32,0.03031040132045746
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,512,32,0.027750399708747864
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,512,16,0.016896000504493712
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,512,8,0.009728000313043595
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,512,16,0.016998399794101716
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,512,8,0.010444799810647965
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,512,4,0.007577600330114365
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,512,4,0.008089599758386612
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,512,2,0.006143999844789505
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,512,2,0.00788159966468811
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,512,1,0.005939200147986412
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,512,1,0.006553599983453751
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,768,64,0.034483200311660765
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,768,128,0.23306241035461425
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,768,128,0.2802687883377075
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,768,64,0.0775168001651764
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,768,32,0.023347200453281404
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,768,16,0.022918400168418885
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,768,32,0.04116480052471161
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,768,16,0.02385919988155365
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,768,8,0.0130048006772995
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,768,8,0.014336000382900237
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,768,4,0.008601599931716919
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,768,4,0.009216000139713288
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,768,2,0.007372800260782242
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,768,2,0.007577600330114365
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,768,1,0.006553599983453751
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,768,1,0.005734400078654289
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,1024,128,0.3042527914047241
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,1024,128,0.3952640056610107
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,1024,64,0.1176576018333435
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,1024,64,0.045465600490570066
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,1024,32,0.02396160066127777
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,1024,32,0.05437440276145935
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,1024,16,0.027852800488471986
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,1024,8,0.01679359972476959
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,1024,16,0.03031040132045746
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,1024,8,0.017203199863433837
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,1024,4,0.009830400347709656
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,1024,4,0.010547199845314026
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,1024,2,0.007676800340414047
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,1024,2,0.008287999778985977
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,1024,1,0.006041600182652473
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,1024,1,0.007168000191450119
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,1536,128,0.4425824165344238
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,1536,128,0.6108160018920898
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,1536,64,0.21954560279846191
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,1536,64,0.27453439235687255
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,1536,32,0.03461120128631592
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,1536,32,0.07772160172462464
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,1536,16,0.02263039946556091
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,1536,16,0.04147199988365173
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,1536,8,0.022937600314617158
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,1536,4,0.013209599256515502
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,1536,8,0.023244799673557283
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,1536,4,0.01443839967250824
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,1536,2,0.008601599931716919
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,1536,2,0.00939520001411438
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,1536,1,0.005734400078654289
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,1536,1,0.007577600330114365
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,2048,128,0.5805056095123291
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,2048,128,0.879206371307373
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,2048,64,0.2910207986831665
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,2048,64,0.3984384059906006
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,2048,32,0.11458560228347778
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,2048,32,0.045875200629234315
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,2048,16,0.024985599517822265
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,2048,16,0.05427200198173523
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,2048,4,0.01669439971446991
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,2048,8,0.03051519989967346
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,2048,8,0.028569599986076354
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,2048,4,0.017100800573825837
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,2048,2,0.009932799637317658
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,2048,1,0.005632000043988228
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,2048,2,0.010649599879980088
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,2048,1,0.007987199723720551
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,3072,128,0.8545408248901367
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,3072,128,1.3908991813659668
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,3072,64,0.42700800895690916
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,3072,64,0.6022143840789795
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,3072,32,0.21381120681762694
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,3072,32,0.27453439235687255
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,3072,16,0.03461120128631592
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,3072,16,0.0779263973236084
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,3072,8,0.022937600314617158
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,3072,8,0.04147199988365173
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,3072,4,0.02272319942712784
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,3072,2,0.013312000036239623
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,3072,2,0.01454080045223236
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,3072,4,0.02385919988155365
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,3072,1,0.010137599706649781
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,3072,1,0.009113600105047226
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,4096,128,1.1307007789611816
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,4096,128,1.904742431640625
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,4096,64,0.5666816234588623
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,4096,32,0.28815360069274903
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,4096,64,0.8450048446655274
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,4096,16,0.04567039906978607
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,4096,32,0.39045119285583496
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,4096,16,0.10833920240402221
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,4096,8,0.024780799448490144
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,4096,8,0.05437440276145935
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,4096,4,0.027856001257896425
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,4096,4,0.03041279911994934
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,4096,2,0.016979199647903443
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,4096,2,0.01720000058412552
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,4096,1,0.007065600156784058
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,4096,1,0.010649599879980088
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,6144,128,1.7084415435791016
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,6144,64,1.3838335990905761
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,6144,128,2.930790328979492
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,6144,32,0.425164794921875
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,6144,64,0.9013216018676757
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,6144,32,0.5999616146087646
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,6144,16,0.2664448022842407
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,6144,16,0.21483519077301025
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,6144,8,0.03543039858341217
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,6144,8,0.07813119888305664
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,6144,4,0.04136959910392761
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,6144,4,0.023449599742889404
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,6144,2,0.01228799968957901
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,6144,1,0.008396799862384795
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,6144,1,0.01443839967250824
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,6144,2,0.023654399812221526
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,8192,64,1.1829279899597167
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,8192,32,0.8336383819580078
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,8192,64,1.8531328201293946
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,8192,128,2.388070487976074
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,8192,128,3.9969791412353515
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,8192,32,0.5921792030334473
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,8192,16,0.3910655975341797
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,8192,16,0.28733439445495607
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,8192,8,0.04157440066337585
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,8192,8,0.11458560228347778
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,8192,4,0.02457599937915802
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,8192,2,0.013823999464511872
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,8192,4,0.053964799642562865
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,8192,2,0.03020800054073334
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,8192,1,0.009523200243711472
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,8192,1,0.017100800573825837
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,12288,16,0.42557439804077146
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,12288,64,1.7904640197753907
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,12288,128,3.4330623626708983
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,12288,32,0.8971487998962402
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,12288,32,1.33570556640625
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,12288,16,0.5946368217468262
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,12288,128,6.154342269897461
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,12288,64,2.8975103378295897
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,12288,4,0.035622400045394895
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,12288,8,0.20981760025024415
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,12288,8,0.29419519901275637
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,12288,4,0.07813119888305664
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,12288,1,0.01268800050020218
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,12288,2,0.04095999896526337
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,12288,1,0.023552000522613525
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,12288,2,0.023347200453281404
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,16384,64,2.2509504318237306
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,16384,32,1.1308256149291993
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,16384,16,0.5644288063049316
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,16384,32,1.8311168670654296
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,16384,64,3.9174144744873045
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,16384,16,0.8120320320129395
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,16384,128,4.572774505615234
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,16384,128,8.254259490966797
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,16384,8,0.28866560459136964
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,16384,8,0.41943039894104006
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,16384,4,0.04116480052471161
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,16384,2,0.02457599937915802
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,16384,4,0.11663360595703125
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,16384,2,0.053452801704406736
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,16384,1,0.01648640036582947
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,16384,1,0.030003198981285097
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,20480,64,2.834227180480957
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,20480,16,0.6974463939666748
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,20480,32,1.408409595489502
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,20480,16,1.0568703651428222
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,20480,32,2.3434240341186525
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,20480,64,4.992819213867188
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,20480,128,6.005452728271484
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,20480,128,10.49200668334961
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,20480,8,0.36034560203552246
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,20480,8,0.5287936210632325
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,20480,4,0.09297919869422913
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,20480,4,0.22261760234832764
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,20480,2,0.03399679958820343
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,20480,2,0.06584320068359376
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,float16,20480,1,0.022937600314617158
TRTLLM,1.0.0,NVIDIA L40S,mla_gen_post,default,fp8,20480,1,0.03635199964046478
