framework,version,device,op_name,kernel_source,mla_dtype,kv_cache_dtype,num_heads,batch_size,isl,tp_size,step,latency
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,1,16,1,0,0.018432000031073887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,1,16,8,0,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,1,16,32,0,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,1,16,2,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,1,16,16,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,1,16,4,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,1,16,64,0,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,1,16,1,0,0.018432000031073887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,1,16,8,0,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,1,16,2,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,1,16,16,0,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,1,16,32,0,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,1,16,4,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,1,32,1,0,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,1,16,64,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,1,32,2,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,1,32,8,0,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,1,32,16,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,1,32,4,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,1,32,32,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,1,32,1,0,0.018432000031073887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,1,32,2,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,1,32,64,0,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,1,32,4,0,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,1,32,8,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,1,32,16,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,1,32,64,0,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,1,32,32,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,1,64,1,0,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,1,64,2,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,1,64,32,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,1,64,4,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,1,64,8,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,1,64,64,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,1,64,16,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,1,64,1,0,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,1,64,2,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,1,64,4,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,1,64,8,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,1,64,32,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,1,64,16,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,1,64,64,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,1,128,1,0,0.021162666380405426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,1,128,4,0,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,1,128,8,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,1,128,16,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,1,128,2,0,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,1,128,32,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,1,128,64,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,1,128,1,0,0.021333334346612293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,1,128,4,0,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,1,128,2,0,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,1,128,32,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,1,128,8,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,1,128,16,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,1,128,64,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,1,256,1,0,0.0363520011305809
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,1,256,2,0,0.027647999425729115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,1,256,4,0,0.022357332209746044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,1,256,8,0,0.020992000897725422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,1,256,4,0,0.02218666672706604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,1,256,16,0,0.020821332931518555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,1,256,64,0,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,1,256,1,0,0.03583999971548716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,1,256,32,0,0.02065066620707512
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,1,256,2,0,0.027647999425729115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,1,256,8,0,0.020992000897725422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,1,256,32,0,0.02065066620707512
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,1,256,64,0,0.02065066620707512
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,1,256,16,0,0.020821332931518555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,1,512,4,0,0.04113066693147024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,1,512,2,0,0.0506879985332489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,1,512,16,0,0.030037333567937214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,1,512,8,0,0.034304000437259674
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,1,512,1,0,0.07765333354473114
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,1,512,32,0,0.0290133332212766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,1,512,1,0,0.07833600044250488
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,1,512,4,0,0.04113066693147024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,1,512,64,0,0.02935466667016347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,1,512,2,0,0.051370665431022644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,1,512,8,0,0.034304000437259674
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,1,512,64,0,0.02918400118748347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,1,512,16,0,0.029866665601730347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,1,512,32,0,0.02918400118748347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,1,1024,2,0,0.12356266379356384
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,1,1024,8,0,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,1,1024,16,0,0.05341866612434387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,1,1024,1,0,0.20736000935236612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,1,1024,4,0,0.07577600081761678
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,1,1024,32,0,0.048298666874567665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,1,1024,64,0,0.04778666794300079
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,1,1024,1,0,0.2053119937578837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,1,1024,4,0,0.07492266595363617
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,1,1024,64,0,0.04795733094215393
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,1,1024,8,0,0.06843733290831248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,1,1024,16,0,0.053247998158137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,1,1024,32,0,0.048298666874567665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,1,1024,2,0,0.12407466769218445
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,1,1536,2,0,0.22528000672658285
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,1,1536,8,0,0.09659733374913533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,1,1536,4,0,0.1418239971001943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,1,1536,1,0,0.3949226538340251
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,1,1536,16,0,0.08123733103275299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,1,1536,64,0,0.06502399841944377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,1,1536,32,0,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,1,1536,2,0,0.22510933876037598
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,1,1536,4,0,0.16486400365829468
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,1,1536,1,0,0.3985066811243693
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,1,1536,32,0,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,1,1536,8,0,0.09710933764775594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,1,1536,16,0,0.08157866696516673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,1,1536,64,0,0.06485333542029063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,1,2048,16,0,0.12322133779525757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,1,2048,8,0,0.13875200351079306
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,1,2048,4,0,0.21657600005467734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,1,2048,2,0,0.3597653309504191
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,1,2048,64,0,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,1,2048,32,0,0.08328533172607422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,1,2048,1,0,0.6533120075861613
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,1,2048,8,0,0.1372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,1,2048,1,0,0.6575786670049032
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,1,2048,16,0,0.12288000186284383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,1,2048,32,0,0.08294400076071422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,1,2048,4,0,0.21606399615605673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,1,2048,2,0,0.36061867078145343
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,1,2048,64,0,0.08226133386294048
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,1,3072,32,0,0.16247466206550598
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,1,3072,64,0,0.11827199657758077
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,1,3072,16,0,0.17971199750900269
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,1,3072,2,0,0.7212373415629069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,1,3072,8,0,0.24490666389465332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,1,3072,4,0,0.4072106679280599
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,1,3072,1,0,1.3525333404541016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,1,3072,8,0,0.24968532721201578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,1,3072,32,0,0.1629866659641266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,1,3072,64,0,0.11776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,1,3072,16,0,0.1790293256441752
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,1,3072,4,0,0.40635732809702557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,1,3072,2,0,0.7253333727518717
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,1,3072,1,0,1.3622612953186035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,1,4096,16,0,0.26743467648824054
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,1,4096,32,0,0.18090667327245077
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,1,4096,8,0,0.3928746779759725
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,1,4096,64,0,0.15547733505566916
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,1,4096,4,0,0.6729386647542318
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,1,4096,2,0,1.2033706506093342
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,1,4096,1,0,2.302464008331299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,1,4096,16,0,0.2677759925524394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,1,4096,64,0,0.15530666708946228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,1,4096,4,0,0.6705493132273356
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,1,4096,2,0,1.2189013163248699
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,1,4096,8,0,0.4002133210500081
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,1,4096,32,0,0.18056533734003702
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,1,6144,16,0,0.4761600097020467
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,1,4096,1,0,2.324992020924886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,1,6144,8,0,0.7819946606953939
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,1,6144,64,0,0.22528000672658285
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,1,6144,4,0,1.374037265777588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,1,6144,32,0,0.3452586730321248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,1,6144,2,0,2.590208053588867
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,1,6144,16,0,0.47598934173583984
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,1,6144,8,0,0.7796053091684977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,1,6144,32,0,0.3457706769307454
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,1,6144,64,0,0.23278933763504028
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,1,6144,4,0,1.3765974044799805
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,1,6144,1,0,5.12768014272054
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,1,6144,2,0,2.5779199600219727
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,1,8192,16,0,0.7847253481547037
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,1,8192,8,0,1.2811946868896484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,1,8192,32,0,0.5084160168965658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,1,8192,64,0,0.3536213239034017
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,1,6144,1,0,5.143552144368489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,1,8192,4,0,2.3386452992757163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,1,8192,2,0,4.411904017130534
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,1,8192,16,0,0.7511040369669596
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,1,8192,32,0,0.5072213411331177
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,1,8192,8,0,1.2887039979298909
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,1,8192,64,0,0.35174401601155597
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,1,8192,4,0,2.3381333351135254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,1,8192,2,0,4.509183883666992
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,1,10240,16,0,1.0943146546681721
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,1,10240,8,0,1.912490685780843
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,1,8192,1,0,8.804351806640625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,1,10240,32,0,0.7145813306172689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,1,10240,4,0,3.5899734497070312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,1,10240,64,0,0.5021013418833414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,1,8192,1,0,8.901290893554688
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,1,10240,16,0,1.110869328180949
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,1,10240,32,0,0.7098026275634766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,1,10240,8,0,1.9211947123209636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,1,10240,4,0,3.5839999516805015
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,1,10240,2,0,6.996480305989583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,1,10240,64,0,0.5021013418833414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,1,10240,2,0,6.892714818318685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,1,12288,8,0,2.67195733388265
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,1,12288,16,0,1.5435093243916829
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,1,10240,1,0,13.946367899576822
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,1,12288,32,0,0.9477120240529379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,1,12288,4,0,5.125290552775065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,1,12288,64,0,0.6505813201268514
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,1,12288,16,0,1.53548796971639
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,1,12288,2,0,9.967615763346354
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,1,10240,1,0,13.569024403889975
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,1,12288,8,0,2.674858729044596
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,1,12288,4,0,5.042688051859538
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,1,12288,32,0,0.9454933007558187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,1,12288,64,0,0.65774933497111
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,1,12288,2,0,10.033493041992188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,1,16384,16,0,2.538837273915609
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,1,16384,8,0,4.612778663635254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,1,16384,32,0,1.4578347206115723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,1,12288,1,0,19.69766362508138
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,1,16384,64,0,1.021781365076701
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,1,16384,4,0,8.78216552734375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,1,12288,1,0,19.749205271402996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,1,16384,2,0,17.677823384602863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,1,16384,8,0,4.630357424418132
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,1,16384,16,0,2.5231359799702964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,2,16,2,0,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,2,16,1,0,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,1,16384,64,0,1.0228053728739421
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,1,16384,32,0,1.4510080019632976
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,1,16384,4,0,8.795989354451498
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,2,16,4,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,2,16,8,0,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,2,16,16,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,2,16,32,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,2,16,64,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,2,16,1,0,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,2,16,2,0,0.018432000031073887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,2,16,4,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,2,16,16,0,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,2,16,8,0,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,2,16,32,0,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,2,32,1,0,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,2,16,64,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,2,32,2,0,0.018432000031073887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,2,32,4,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,1,16384,2,0,17.672021230061848
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,2,32,8,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,2,32,32,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,2,32,16,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,2,32,1,0,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,2,32,64,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,2,32,2,0,0.018602666755517323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,2,32,4,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,2,32,8,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,2,32,16,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,1,16384,1,0,34.148180643717446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,2,32,32,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,2,32,64,0,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,2,64,1,0,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,2,64,2,0,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,2,64,8,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,2,64,4,0,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,2,64,16,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,2,64,32,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,2,64,64,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,2,64,4,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,2,64,8,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,2,64,2,0,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,2,64,1,0,0.02065066620707512
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,2,64,32,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,2,64,16,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,2,64,64,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,2,128,1,0,0.028501334289709728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,2,128,2,0,0.021162666380405426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,2,128,4,0,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,2,128,16,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,2,128,8,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,2,128,64,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,2,128,32,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,2,128,1,0,0.0290133332212766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,2,128,2,0,0.021162666380405426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,2,128,4,0,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,2,128,8,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,2,128,32,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,1,16384,1,0,34.74414825439453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,2,128,64,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,2,128,16,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,2,256,2,0,0.03601066768169403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,2,256,4,0,0.027989332874615986
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,2,256,8,0,0.02218666672706604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,2,256,1,0,0.058880001306533813
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,2,256,16,0,0.020821332931518555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,2,256,32,0,0.020821332931518555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,2,256,64,0,0.02065066620707512
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,2,256,2,0,0.03669333209594091
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,2,256,4,0,0.027477333943049114
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,2,256,8,0,0.02218666672706604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,2,256,16,0,0.020992000897725422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,2,256,1,0,0.058880001306533813
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,2,256,32,0,0.020821332931518555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,2,256,64,0,0.02065066620707512
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,2,512,1,0,0.12970667084058127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,2,512,2,0,0.09983999530474345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,2,512,8,0,0.04164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,2,512,16,0,0.034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,2,512,64,0,0.029525332152843475
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,2,512,4,0,0.0506879985332489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,2,512,32,0,0.03054933249950409
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,2,512,1,0,0.1293653349081675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,2,512,2,0,0.07901866734027863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,2,512,8,0,0.04164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,2,512,4,0,0.05034666756788889
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,2,512,16,0,0.034645333886146545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,2,512,32,0,0.03054933249950409
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,2,512,64,0,0.029525332152843475
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,2,1024,2,0,0.2058239976565043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,2,1024,8,0,0.07765333354473114
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,2,1024,16,0,0.06877866884072621
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,2,1024,32,0,0.05376000205675761
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,2,1024,64,0,0.048298666874567665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,2,1024,4,0,0.12322133779525757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,2,1024,1,0,0.3805866638819377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,2,1024,4,0,0.12424533565839131
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,2,1024,8,0,0.07901866734027863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,2,1024,16,0,0.0682666649421056
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,2,1024,32,0,0.05376000205675761
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,2,1024,1,0,0.3768320083618164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,2,1024,2,0,0.20667733748753866
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,2,1024,64,0,0.048469334840774536
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,2,1536,4,0,0.226474662621816
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,2,1536,8,0,0.14148267110188803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,2,1536,16,0,0.0981333355108897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,2,1536,32,0,0.0820906658967336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,2,1536,2,0,0.4031146764755249
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,2,1536,64,0,0.067071999112765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,2,1536,1,0,0.7589546839396158
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,2,1536,8,0,0.14148267110188803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,2,1536,32,0,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,2,1536,4,0,0.22715733448664346
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,2,1536,64,0,0.067071999112765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,2,1536,16,0,0.09847467144330342
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,2,1536,2,0,0.40174933274586994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,2,1536,1,0,0.7519573370615641
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,2,2048,32,0,0.12458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,2,2048,64,0,0.08430932958920796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,2,2048,4,0,0.3638613224029541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,2,2048,16,0,0.1389226714769999
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,2,2048,8,0,0.2177706758181254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,2,2048,2,0,0.65774933497111
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,2,2048,1,0,1.2569599946339924
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,2,2048,32,0,0.12407466769218445
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,2,2048,64,0,0.08413867155710857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,2,2048,8,0,0.21623466412226358
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,2,2048,16,0,0.1397760013739268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,2,2048,4,0,0.362496018409729
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,2,2048,2,0,0.6579200029373169
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,2,2048,1,0,1.2528639634450276
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,2,3072,16,0,0.24883200724919638
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,2,3072,32,0,0.1853440006573995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,2,3072,8,0,0.41045331954956055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,2,3072,64,0,0.16537599762280783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,2,3072,4,0,0.7261866728464762
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,2,3072,16,0,0.24490666389465332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,2,3072,32,0,0.18449066082636514
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,2,3072,8,0,0.41523198286692303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,2,3072,2,0,1.3624320030212402
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,2,3072,4,0,0.7357439994812012
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,2,3072,64,0,0.16452266772588095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,2,3072,2,0,1.358847935994466
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,2,3072,1,0,2.634239991505941
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,2,4096,8,0,0.6833493709564209
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,2,4096,16,0,0.4073813358942668
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,2,4096,4,0,1.224021355311076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,2,4096,32,0,0.2725546757380168
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,2,4096,64,0,0.18978132804234824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,2,3072,1,0,2.618197282155355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,2,4096,16,0,0.4077226718266805
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,2,4096,8,0,0.661845326423645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,2,4096,2,0,2.3217493693033853
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,2,4096,32,0,0.27153066794077557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,2,4096,4,0,1.2260693709055583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,2,4096,64,0,0.18653867642084757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,2,4096,1,0,4.641109466552734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,2,4096,2,0,2.313216050465902
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,2,6144,16,0,0.7869439919789633
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,2,6144,8,0,1.3834239641825359
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,2,6144,32,0,0.49032533168792725
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,2,6144,64,0,0.35259731610616046
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,2,6144,4,0,2.5854293505350747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,2,4096,1,0,4.639744122823079
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,2,6144,16,0,0.785919984181722
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,2,6144,32,0,0.4962986707687378
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,2,6144,8,0,1.377962589263916
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,2,6144,2,0,5.0430294672648115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,2,6144,64,0,0.353279987970988
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,2,6144,4,0,2.5758719444274902
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,2,6144,2,0,5.1123199462890625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,2,8192,16,0,1.2950186729431152
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,2,8192,8,0,2.32806396484375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,2,6144,1,0,10.13913599650065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,2,8192,64,0,0.5152426560719808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,2,8192,32,0,0.7741440137227377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,2,8192,4,0,4.5136213302612305
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,2,6144,1,0,10.254677454630533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,2,8192,16,0,1.300650676091512
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,2,8192,8,0,2.3575894037882485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,2,8192,2,0,8.866474787394205
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,2,8192,64,0,0.5213866631189982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,2,8192,4,0,4.535978635152181
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,2,8192,32,0,0.7787520090738932
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,2,10240,16,0,1.9343360265096028
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,2,8192,2,0,8.963071823120117
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,2,10240,8,0,3.5846827824910483
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,2,10240,32,0,1.1197439829508464
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,2,10240,64,0,0.7352320353190104
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,2,8192,1,0,17.703253428141277
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,2,10240,4,0,6.9705384572347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,2,10240,16,0,1.9309226671854656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,2,10240,8,0,3.5703465143839517
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,2,10240,2,0,13.640191396077475
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,2,10240,32,0,1.0987520217895508
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,2,8192,1,0,17.48633575439453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,2,10240,64,0,0.730282704035441
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,2,10240,4,0,7.06440544128418
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,2,12288,16,0,2.706261316935221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,2,12288,8,0,5.075968106587728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,2,10240,2,0,13.953535715738932
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,2,12288,32,0,1.5450453758239746
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,2,12288,64,0,0.9741653601328532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,2,12288,4,0,9.765375773111979
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,2,10240,1,0,27.579050699869793
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,2,12288,16,0,2.7095041275024414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,2,12288,8,0,5.030741373697917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,2,12288,64,0,0.9632426897684733
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,2,12288,32,0,1.5556267102559407
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,2,12288,2,0,19.691519419352215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,2,12288,4,0,9.945088068644205
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,2,10240,1,0,27.575124104817707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,2,16384,16,0,4.6619307200113935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,2,12288,2,0,19.911168416341145
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,2,16384,8,0,8.980991999308268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,2,16384,64,0,1.478314717610677
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,2,16384,32,0,2.5332053502400718
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,2,16384,4,0,17.543168385823567
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,2,12288,1,0,38.93384552001953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,2,16384,16,0,4.574890772501628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,2,16384,8,0,8.923477172851562
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,2,16384,4,0,17.735679626464844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,2,16384,32,0,2.559658686319987
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,2,16384,64,0,1.4825812975565593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,2,12288,1,0,39.325696309407554
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,4,16,1,0,0.025087999800841015
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,2,16384,2,0,34.90935516357422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,4,16,2,0,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,4,16,4,0,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,4,16,8,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,4,16,16,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,4,16,32,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,4,16,64,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,4,16,1,0,0.025258667767047882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,4,16,2,0,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,4,16,4,0,0.018090666582187016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,4,16,8,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,4,16,16,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,4,16,32,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,4,16,64,0,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,4,32,1,0,0.025941332181294758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,4,32,8,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,4,32,4,0,0.018432000031073887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,4,32,2,0,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,4,32,16,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,4,32,32,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,4,32,2,0,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,4,32,64,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,4,32,4,0,0.018602666755517323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,4,32,8,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,4,32,1,0,0.025770666698614757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,4,32,16,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,4,32,32,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,4,32,64,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,2,16384,2,0,35.15101877848307
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,4,64,1,0,0.027647999425729115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,4,64,2,0,0.020309332758188248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,4,64,4,0,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,4,64,8,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,4,64,16,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,4,64,32,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,4,64,64,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,4,64,1,0,0.027136000494162243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,4,64,8,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,4,64,2,0,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,4,64,4,0,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,4,64,16,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,4,64,32,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,4,64,64,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,4,128,1,0,0.051029334465662636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,4,128,8,0,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,4,128,16,0,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,4,128,4,0,0.02184533327817917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,4,128,32,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,4,128,2,0,0.028501334289709728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,4,128,64,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,4,128,4,0,0.021162666380405426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,4,128,1,0,0.051029334465662636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,4,128,16,0,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,4,128,8,0,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,4,128,2,0,0.02867199977238973
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,4,128,32,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,4,128,64,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,4,256,16,0,0.022357332209746044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,4,256,8,0,0.028330666323502857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,4,256,32,0,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,4,256,4,0,0.037205333511034645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,4,256,1,0,0.09762133161226909
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,4,256,2,0,0.05870933334032694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,4,256,64,0,0.021333334346612293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,2,16384,1,0,68.64469401041667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,4,256,2,0,0.059392000238100685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,4,256,16,0,0.022357332209746044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,4,256,8,0,0.028330666323502857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,4,256,4,0,0.0365226666132609
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,4,256,1,0,0.09693866968154907
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,4,256,32,0,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,4,256,64,0,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,4,512,16,0,0.04164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,4,512,32,0,0.03498666733503342
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,4,512,4,0,0.07901866734027863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,4,512,8,0,0.05273599922657013
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,4,512,64,0,0.03089066594839096
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,4,512,2,0,0.5072213411331177
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,2,16384,1,0,69.99978637695312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,4,512,1,0,0.24524799982706705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,4,512,4,0,0.07918933530648549
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,4,512,8,0,0.05205333232879639
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,4,512,16,0,0.04181333382924398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,4,512,32,0,0.034815999368826546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,4,512,64,0,0.0314026673634847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,4,512,2,0,0.13124266266822815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,4,512,1,0,0.2455893357594808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,4,1024,8,0,0.1256106694539388
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,4,1024,16,0,0.07884799937407176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,4,1024,64,0,0.055125330885251365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,4,1024,32,0,0.06997333467006683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,4,1024,4,0,0.20957867304484049
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,4,1024,2,0,0.3831466833750407
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,4,1024,16,0,0.07935999830563863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,4,1024,32,0,0.06980266670385997
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,4,1024,1,0,0.7350613276163737
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,4,1024,8,0,0.1262933313846588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,4,1024,4,0,0.20838399728139242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,4,1024,64,0,0.05529599885145823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,4,1024,2,0,0.3860479990641276
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,4,1024,1,0,0.7309652964274088
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,4,1536,16,0,0.14882133404413858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,4,1536,64,0,0.08533333738644917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,4,1536,32,0,0.10205866893132527
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,4,1536,4,0,0.4106239875157674
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,4,1536,8,0,0.22920533021291098
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,4,1536,2,0,0.7604906558990479
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,4,1536,16,0,0.15001599987347922
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,4,1536,32,0,0.10240000486373901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,4,1536,8,0,0.23091200987497965
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,4,1536,4,0,0.4060159921646118
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,4,1536,64,0,0.08413867155710857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,4,1536,1,0,1.4743893941243489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,4,1536,2,0,0.7569066683451334
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,4,2048,8,0,0.3643733263015747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,4,2048,16,0,0.22323199113210043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,4,2048,4,0,0.6676479975382487
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,4,2048,32,0,0.1462613344192505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,4,1536,1,0,1.4692692756652832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,4,2048,64,0,0.12970667084058127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,4,2048,2,0,1.2598613103230794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,4,2048,16,0,0.2198186715443929
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,4,2048,32,0,0.14404267072677612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,4,2048,8,0,0.366592009862264
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,4,2048,64,0,0.1293653349081675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,4,2048,4,0,0.66594131787618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,4,2048,1,0,2.456746737162272
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,4,2048,2,0,1.2620800336201985
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,4,3072,16,0,0.4271786610285441
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,4,3072,8,0,0.7377920150756836
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,4,3072,4,0,1.3757440249125164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,4,3072,32,0,0.2599253257115682
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,4,3072,64,0,0.19353600343068442
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,4,2048,1,0,2.448042710622152
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,4,3072,2,0,2.651306629180908
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,4,3072,16,0,0.4193280140558879
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,4,3072,8,0,0.7406933307647705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,4,3072,32,0,0.2583893338839213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,4,3072,4,0,1.3813759485880535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,4,3072,64,0,0.19473065932591757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,4,3072,2,0,2.6333866119384766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,4,4096,16,0,0.6917119820912679
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,4,4096,8,0,1.2340906461079915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,4,4096,32,0,0.4072106679280599
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,4,4096,64,0,0.28074665864308673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,4,3072,1,0,5.458431879679362
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,4,4096,4,0,2.333354632059733
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,4,3072,1,0,5.392896016438802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,4,4096,16,0,0.6835199991861979
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,4,4096,2,0,4.605098724365234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,4,4096,8,0,1.2330666383107503
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,4,4096,32,0,0.4140373468399048
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,4,4096,64,0,0.28177066644032794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,4,4096,4,0,2.322773297627767
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,4,6144,16,0,1.4049280484517415
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,4,4096,2,0,4.675413449605306
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,4,6144,8,0,2.604032039642334
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,4,6144,64,0,0.504319985707601
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,4,4096,1,0,9.2315305074056
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,4,6144,32,0,0.816810687383016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,4,6144,4,0,5.1858774820963545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,4,4096,1,0,9.246378580729166
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,4,6144,16,0,1.3958826065063477
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,4,6144,8,0,2.5941333770751953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,4,6144,2,0,10.252117156982422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,4,6144,64,0,0.4968106746673584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,4,6144,32,0,0.8279039859771729
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,4,6144,4,0,5.065727869669597
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,4,8192,16,0,2.364586671193441
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,4,8192,8,0,4.573184013366699
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,4,6144,2,0,10.262016296386719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,4,8192,32,0,1.3173759778340657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,4,8192,64,0,0.8034986654917399
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,4,8192,4,0,8.920576095581055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,4,6144,1,0,20.082687377929688
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,4,8192,8,0,4.465152104695638
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,4,8192,16,0,2.370901266733805
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,4,8192,2,0,17.80309295654297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,4,6144,1,0,19.91185124715169
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,4,8192,64,0,0.7864320278167725
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,4,8192,32,0,1.3195947011311848
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,4,8192,4,0,8.989866892496744
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,8,16,1,0,0.034645333886146545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,8,16,2,0,0.025429333249727886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,8,16,4,0,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,8,16,16,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,8,16,8,0,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,8,16,32,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,8,16,64,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,8,16,1,0,0.03515733281771342
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,8,16,2,0,0.025258667767047882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,8,16,4,0,0.01877333347996076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,8,16,8,0,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,8,16,16,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,8,16,32,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,8,16,64,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,8,32,1,0,0.03618133316437403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,8,32,2,0,0.02611200014750163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,8,32,8,0,0.01877333347996076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,8,32,4,0,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,8,32,16,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,8,32,64,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,8,32,32,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,8,32,1,0,0.03566933423280716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,8,32,4,0,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,8,32,2,0,0.025941332181294758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,8,32,8,0,0.018432000031073887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,8,32,16,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,8,32,32,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,4,8192,2,0,17.582421620686848
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,8,32,64,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,8,64,2,0,0.027477333943049114
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,8,64,4,0,0.020821332931518555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,8,64,8,0,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,8,64,32,0,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,8,64,16,0,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,8,64,64,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,8,64,1,0,0.050517335534095764
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,8,64,2,0,0.027477333943049114
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,8,64,1,0,0.05017599960168203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,8,64,4,0,0.020821332931518555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,8,64,8,0,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,8,64,16,0,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,8,64,32,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,8,64,64,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,8,128,4,0,0.029696000119050343
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,8,128,16,0,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,8,128,2,0,0.051541333397229515
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,8,128,32,0,0.01672533278663953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,8,128,8,0,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,8,128,1,0,0.09318400422732036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,4,8192,1,0,35.748181660970054
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,8,128,64,0,0.016554666062196095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,8,128,16,0,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,8,128,8,0,0.02184533327817917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,8,128,4,0,0.029866665601730347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,8,128,2,0,0.05120000243186951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,8,128,32,0,0.016895999511082966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,8,128,64,0,0.016554666062196095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,8,128,1,0,0.1129813293615977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,8,256,4,0,0.06058666606744131
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,8,256,8,0,0.03822933385769526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,8,256,16,0,0.028330666323502857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,8,256,2,0,0.09847467144330342
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,8,256,64,0,0.022015998760859173
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,8,256,32,0,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,8,256,1,0,0.18158932526906332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,8,256,4,0,0.06058666606744131
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,8,256,8,0,0.03925333420435587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,8,256,16,0,0.028160000840822857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,8,256,32,0,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,8,256,64,0,0.02218666672706604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,8,256,2,0,0.09864532947540283
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,8,256,1,0,0.18244266510009766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,8,512,16,0,0.05461333195368449
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,8,512,32,0,0.042837331692377724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,8,512,64,0,0.03601066768169403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,8,512,8,0,0.08106666803359985
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,8,512,4,0,0.13687466581662497
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,8,512,2,0,0.2474666635195414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,8,512,16,0,0.054101333022117615
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,8,512,8,0,0.08089600006739299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,8,512,32,0,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,8,512,1,0,0.47274665037790936
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,8,512,4,0,0.13550933202107748
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,4,8192,1,0,35.45736440022787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,8,512,2,0,0.2491733431816101
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,8,512,64,0,0.0363520011305809
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,8,512,1,0,0.47223468621571857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,8,1024,16,0,0.130730668703715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,8,1024,32,0,0.08430932958920796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,8,1024,8,0,0.21538132429122925
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,8,1024,64,0,0.07526400188604991
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,8,1024,4,0,0.3938986857732137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,8,1024,16,0,0.12970667084058127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,8,1024,8,0,0.21742933988571167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,8,1024,32,0,0.08618666728337605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,8,1024,2,0,0.7393279870351156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,8,1024,4,0,0.3940693140029907
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,8,1024,64,0,0.07543466488520305
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,8,1024,2,0,0.7383039792378744
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,8,1536,8,0,0.41915734608968097
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,8,1536,16,0,0.2397866646448771
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,8,1024,1,0,1.4341120719909668
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,8,1536,32,0,0.1525759994983673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,8,1536,4,0,0.7727786699930826
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,8,1536,64,0,0.1104213297367096
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,8,1024,1,0,1.4361599286397297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,8,1536,32,0,0.15428266922632852
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,8,1536,2,0,1.4769493738810222
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,8,1536,8,0,0.4148906469345093
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,8,1536,16,0,0.23961599667867026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,8,1536,4,0,0.7705600261688232
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,8,1536,64,0,0.1104213297367096
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,8,1536,2,0,1.4759252866109211
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,8,2048,16,0,0.37990399201711017
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,8,2048,8,0,0.680618683497111
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,8,2048,4,0,1.279317299524943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,8,2048,32,0,0.23142399390538534
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,8,2048,64,0,0.15684266885121664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,8,1536,1,0,2.9279572168986
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,8,2048,16,0,0.38690133889516193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,8,1536,1,0,2.8888746897379556
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,8,2048,8,0,0.6795946756998698
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,8,2048,2,0,2.4644266764322915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,8,2048,64,0,0.15837867061297098
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,8,2048,32,0,0.23586134115854898
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,8,2048,4,0,1.275221347808838
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,8,3072,16,0,0.7729492982228597
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,8,2048,2,0,2.4702293078104653
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,8,3072,8,0,1.3895680109659831
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,8,3072,64,0,0.27084799607594806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,8,3072,32,0,0.4375893274943034
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,8,2048,1,0,5.066752115885417
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,8,3072,4,0,2.6552319526672363
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,8,2048,1,0,5.073919932047526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,8,3072,16,0,0.7524693012237549
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,8,3072,8,0,1.3938345909118652
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,8,3072,2,0,5.389141082763672
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,8,3072,32,0,0.44066135088602704
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,8,3072,64,0,0.27033599217732746
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,8,3072,4,0,2.655061403910319
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,8,4096,16,0,1.2581546306610107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,8,4096,8,0,2.337792078653971
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,8,3072,2,0,5.313536008199056
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,8,4096,32,0,0.7067306836446127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,8,4096,64,0,0.4532906611760457
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,8,4096,4,0,4.68667729695638
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,8,3072,1,0,10.843477884928385
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,8,4096,16,0,1.2646400133768718
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,8,4096,8,0,2.3500800132751465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,8,4096,2,0,9.380010604858398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,8,3072,1,0,10.828970591227213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,8,4096,64,0,0.44407467047373456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,8,4096,32,0,0.7077546914418539
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,16,16,1,0,0.05717333157857259
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,8,4096,4,0,4.55185063680013
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,16,16,8,0,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,16,16,4,0,0.025770666698614757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,16,16,2,0,0.03515733281771342
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,16,16,16,0,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,16,16,32,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,16,16,64,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,16,16,1,0,0.056832000613212585
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,16,16,4,0,0.025087999800841015
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,16,16,2,0,0.03515733281771342
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,16,16,8,0,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,16,16,16,0,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,16,16,32,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,16,16,64,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,16,32,1,0,0.06365866462389629
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,16,32,2,0,0.03618133316437403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,16,32,8,0,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,16,32,4,0,0.02628266563018163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,16,32,16,0,0.018602666755517323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,16,32,32,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,16,32,64,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,8,4096,2,0,9.264810562133789
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,16,32,4,0,0.02611200014750163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,16,32,2,0,0.0365226666132609
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,16,32,8,0,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,16,32,1,0,0.06348800162474315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,16,32,16,0,0.018432000031073887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,16,32,32,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,16,32,64,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,16,64,4,0,0.028330666323502857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,16,64,2,0,0.0506879985332489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,16,64,8,0,0.021162666380405426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,16,64,1,0,0.08823466300964355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,16,64,16,0,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,16,64,32,0,0.01672533278663953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,16,64,64,0,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,16,64,16,0,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,16,64,4,0,0.028160000840822857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,16,64,2,0,0.0506879985332489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,16,64,32,0,0.016554666062196095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,16,64,1,0,0.08789333701133728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,16,64,8,0,0.021333334346612293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,16,64,64,0,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,16,128,8,0,0.0314026673634847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,16,128,16,0,0.022015998760859173
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,16,128,4,0,0.053077335158983864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,16,128,32,0,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,16,128,2,0,0.09198932846387227
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,16,128,1,0,0.17015467087427774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,16,128,64,0,0.017407999684413273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,8,4096,1,0,18.73629887898763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,16,128,4,0,0.052906667192777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,16,128,2,0,0.09215999643007915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,16,128,8,0,0.0314026673634847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,16,128,32,0,0.020821332931518555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,16,128,16,0,0.022357332209746044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,16,128,64,0,0.017407999684413273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,16,128,1,0,0.16964266697565714
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,16,256,16,0,0.040789333482583366
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,16,256,8,0,0.06331733365853627
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,16,256,4,0,0.10308266679445903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,16,256,32,0,0.029525332152843475
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,16,256,64,0,0.024234667420387268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,16,256,2,0,0.1853440006573995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,16,256,16,0,0.04164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,16,256,8,0,0.06348800162474315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,16,256,4,0,0.10308266679445903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,16,256,1,0,0.35174401601155597
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,16,256,32,0,0.029525332152843475
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,16,256,2,0,0.1853440006573995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,16,256,64,0,0.0240639994541804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,8,4096,1,0,18.361343383789062
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,16,256,1,0,0.3515733480453491
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,16,512,16,0,0.08686932921409607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,16,512,32,0,0.05973333120346069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,16,512,64,0,0.04983466863632202
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,16,512,8,0,0.1418239971001943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,16,512,4,0,0.2578773299853007
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,16,512,32,0,0.059562668204307556
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,16,512,16,0,0.0865280032157898
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,16,512,8,0,0.14148267110188803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,16,512,2,0,0.4797439972559611
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,16,512,4,0,0.2577066620190938
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,16,512,64,0,0.048810665806134544
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,16,512,2,0,0.4805973370869954
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,16,512,1,0,0.9282560348510742
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,16,1024,16,0,0.23040000597635904
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,16,1024,8,0,0.4041386842727661
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,16,1024,32,0,0.14404267072677612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,16,1024,64,0,0.09489066402117412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,16,512,1,0,0.928767999013265
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,16,1024,4,0,0.7569066683451334
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,16,1024,32,0,0.1443839967250824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,16,1024,16,0,0.23227733373641968
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,16,1024,2,0,1.4525440533955891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,16,1024,8,0,0.4002133210500081
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,16,1024,64,0,0.09540266791979472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,16,1024,4,0,0.7596373558044434
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,16,1536,16,0,0.4346880118052165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,16,1024,2,0,1.4510080019632976
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,16,1536,8,0,0.7855786482493082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,16,1536,32,0,0.2590720057487488
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,16,1536,64,0,0.17612799008687338
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,16,1024,1,0,2.8779519399007163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,16,1536,4,0,1.503061294555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,16,1536,16,0,0.43485867977142334
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,16,1024,1,0,2.877610524495443
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,16,1536,8,0,0.7867733637491862
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,16,1536,2,0,2.9438292185465493
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,16,1536,64,0,0.17425066232681274
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,16,1536,32,0,0.2599253257115682
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,16,1536,4,0,1.5018666585286458
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,16,2048,16,0,0.6973439852396647
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,16,2048,8,0,1.293994665145874
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,16,1536,2,0,2.9491198857625327
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,16,2048,32,0,0.40328534444173175
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,16,2048,64,0,0.2578773299853007
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,16,2048,4,0,2.4932692845662436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,16,1536,1,0,6.024362564086914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,16,2048,16,0,0.7080960273742676
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,16,2048,8,0,1.2948479652404785
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,16,2048,2,0,5.06658140818278
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,16,2048,32,0,0.40567465623219806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,16,2048,64,0,0.2558293342590332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,16,1536,1,0,5.919573465983073
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,16,2048,4,0,2.5077759424845376
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,32,16,2,0,0.0576853354771932
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,32,16,1,0,0.10444800059000652
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,32,16,4,0,0.03532800078392029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,32,16,8,0,0.025600001215934753
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,32,16,16,0,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,32,16,32,0,0.018602666755517323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,32,16,64,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,32,16,2,0,0.057002668579419456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,32,16,1,0,0.10410666465759277
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,32,16,4,0,0.03549866626660029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,32,16,8,0,0.025600001215934753
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,32,16,16,0,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,32,16,32,0,0.018432000031073887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,32,16,64,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,32,32,4,0,0.036864000062147774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,32,32,2,0,0.06468266745408376
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,32,32,16,0,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,32,32,8,0,0.026965332527955372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,32,32,1,0,0.11400533715883891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,32,32,32,0,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,32,32,64,0,0.016554666062196095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,32,32,1,0,0.11366400122642517
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,32,32,4,0,0.03703466554482778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,32,32,2,0,0.06434133152167003
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,32,32,8,0,0.027136000494162243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,32,32,16,0,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,16,2048,2,0,4.993365287780762
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,32,32,64,0,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,32,32,32,0,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,32,64,4,0,0.051882664362589516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,32,64,2,0,0.08959999680519104
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,32,64,32,0,0.020309332758188248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,32,64,8,0,0.030207999050617218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,32,64,16,0,0.021674667795499165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,32,64,64,0,0.017407999684413273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,32,64,1,0,0.16708266735076904
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,32,64,8,0,0.029525332152843475
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,32,64,4,0,0.051882664362589516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,32,64,16,0,0.021674667795499165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,32,64,2,0,0.0890880028406779
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,32,64,32,0,0.020309332758188248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,32,64,64,0,0.01757866640885671
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,32,64,1,0,0.16913066307703653
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,32,128,8,0,0.05614933371543884
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,32,128,4,0,0.09676800171534221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,32,128,16,0,0.03515733281771342
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,32,128,32,0,0.02372266600529353
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,16,2048,1,0,10.229589462280273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,32,128,2,0,0.1723733345667521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,32,128,64,0,0.02184533327817917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,32,128,8,0,0.05649066468079885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,32,128,16,0,0.03583999971548716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,32,128,1,0,0.3275093237559001
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,32,128,4,0,0.09642666578292847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,32,128,32,0,0.023381332556406658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,32,128,2,0,0.17339734236399332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,32,128,64,0,0.022015998760859173
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,32,128,1,0,0.3269973397254944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,32,256,8,0,0.1109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,32,256,32,0,0.06690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,32,256,16,0,0.06946133573849995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,32,256,4,0,0.1950719952583313
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,32,256,64,0,0.03549866626660029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,16,2048,1,0,10.141525268554688
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,32,256,2,0,0.3590826590855916
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,32,256,8,0,0.11025066177050273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,32,256,16,0,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,32,256,32,0,0.04966400067011515
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,32,256,4,0,0.1955839991569519
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,32,256,64,0,0.03515733281771342
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,32,256,2,0,0.35942399501800537
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,32,256,1,0,0.6908586819966634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,32,512,16,0,0.1534293293952942
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,32,512,8,0,0.26828799645106
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,32,512,32,0,0.11178666353225708
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,32,512,64,0,0.07321600119272868
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,32,512,4,0,0.4983466863632202
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,32,256,1,0,0.6923946539560953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,32,512,8,0,0.265557328859965
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,32,512,32,0,0.09847467144330342
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,32,512,16,0,0.15803733468055725
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,32,512,2,0,0.9425919850667318
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,32,512,4,0,0.49766401449839276
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,32,512,64,0,0.0721919983625412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,32,512,2,0,0.9448106288909912
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,32,1024,16,0,0.4253013531366984
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,32,1024,8,0,0.7755093574523926
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,32,1024,64,0,0.16622933745384216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,32,1024,32,0,0.25497599442799884
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,32,512,1,0,1.8472960789998372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,32,1024,4,0,1.4766079584757488
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,32,512,1,0,1.8428586324055989
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,32,1024,8,0,0.7732906341552734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,32,1024,16,0,0.4288853406906128
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,32,1024,32,0,0.2553173303604126
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,64,16,1,0,0.19473065932591757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,32,1024,64,0,0.16401066382726034
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,64,16,2,0,0.10564266641934712
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,32,1024,4,0,1.481045405069987
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,64,16,4,0,0.05853866537412008
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,64,16,8,0,0.03618133316437403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,64,16,16,0,0.02628266563018163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,64,16,64,0,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,64,16,32,0,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,64,16,1,0,0.19524266322453818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,64,16,2,0,0.10496000448862712
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,64,16,4,0,0.0576853354771932
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,64,16,8,0,0.03532800078392029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,32,1024,2,0,2.9062827428181968
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,64,16,16,0,0.02611200014750163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,64,16,32,0,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,64,16,64,0,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,64,32,16,0,0.027477333943049114
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,64,32,8,0,0.03857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,64,32,4,0,0.06604800124963124
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,64,32,1,0,0.2218666672706604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,64,32,2,0,0.11724799871444702
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,64,32,32,0,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,64,32,64,0,0.019968000551064808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,32,1024,2,0,2.8984320958455405
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,64,32,8,0,0.03857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,64,32,2,0,0.11707733074824016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,64,32,4,0,0.06604800124963124
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,64,32,16,0,0.027477333943049114
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,64,32,1,0,0.2310826579729716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,64,32,64,0,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,64,32,32,0,0.020821332931518555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,64,64,8,0,0.054272000988324486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,64,64,16,0,0.03379199902216593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,64,64,32,0,0.023039999107519787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,64,64,4,0,0.09437867005666097
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,64,64,64,0,0.021674667795499165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,64,64,2,0,0.1718613306681315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,64,64,8,0,0.053930665055910744
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,64,64,4,0,0.09301333626111348
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,64,64,16,0,0.03362133353948593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,64,64,2,0,0.17373865842819214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,64,64,1,0,0.3256319959958394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,64,64,32,0,0.023039999107519787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,64,64,64,0,0.021162666380405426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,64,64,1,0,0.32682667175928753
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,32,1024,1,0,5.8147837320963545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,64,128,16,0,0.06229333579540253
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,64,128,32,0,0.04369066655635834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,64,128,8,0,0.1013759970664978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,64,128,4,0,0.183296004931132
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,64,128,64,0,0.03054933249950409
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,64,128,2,0,0.33484800656636554
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,64,128,8,0,0.1013759970664978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,64,128,16,0,0.0631466656923294
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,64,128,32,0,0.04369066655635834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,64,128,4,0,0.18312533696492514
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,64,128,64,0,0.02867199977238973
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,64,128,2,0,0.33604268232981366
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,32,1024,1,0,5.969237645467122
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,64,128,1,0,0.6452906529108683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,64,256,16,0,0.12475732962290446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,64,256,32,0,0.08584533135096233
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,64,256,64,0,0.06263466676076253
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,64,256,8,0,0.20804266134897867
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,64,128,1,0,0.6439253489176432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,64,256,4,0,0.37546666463216144
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,64,256,32,0,0.08482133348782857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,64,256,16,0,0.12339199582735698
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,64,256,8,0,0.20804266134897867
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,64,256,2,0,0.7063893477121989
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,64,256,64,0,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,64,256,4,0,0.37939198811848956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,64,256,2,0,0.7053653399149576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,64,512,16,0,0.2913279930750529
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,64,512,8,0,0.5201919873555502
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,64,512,64,0,0.12219732999801636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,64,256,1,0,1.3687465985616047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,64,512,32,0,0.1771519978841146
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,64,512,4,0,0.974506696065267
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,64,256,1,0,1.3675519625345867
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,64,512,8,0,0.51746133963267
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,64,512,16,0,0.29286400477091473
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,64,512,64,0,0.12219732999801636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,64,512,32,0,0.18056533734003702
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,64,512,2,0,1.876479943593343
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,128,16,1,0,0.3839999834696452
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,64,512,4,0,0.9757013320922852
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,128,16,4,0,0.10786133011182149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,128,16,2,0,0.19797333081563315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,128,16,16,0,0.036864000062147774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,128,16,32,0,0.027136000494162243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,128,16,8,0,0.06075733403364817
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,128,16,64,0,0.020138667275508244
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,128,16,4,0,0.10734933614730835
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,128,16,16,0,0.0365226666132609
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,128,16,8,0,0.05973333120346069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,128,16,2,0,0.19882667064666748
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,128,16,1,0,0.42871467272440594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,128,16,32,0,0.026965332527955372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,128,16,64,0,0.020309332758188248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,128,32,4,0,0.11963733037312825
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,128,32,16,0,0.042837331692377724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,128,32,8,0,0.06894933183987935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,128,32,2,0,0.22732800245285034
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,64,512,2,0,1.8812586466471355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,128,32,32,0,0.0290133332212766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,128,32,64,0,0.02184533327817917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,128,32,1,0,0.44066135088602704
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,128,32,16,0,0.04215466479460398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,128,32,4,0,0.12134400010108948
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,128,32,8,0,0.06877866884072621
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,128,32,2,0,0.22732800245285034
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,128,32,32,0,0.02935466667016347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,128,32,64,0,0.02184533327817917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,128,32,1,0,0.4398080110549927
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,128,64,16,0,0.062122667829195656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,128,64,4,0,0.1800533334414164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,128,64,8,0,0.09966933727264404
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,128,64,32,0,0.042837331692377724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,128,64,64,0,0.0288426677385966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,128,64,2,0,0.3357013463973999
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,64,512,1,0,3.7060267130533853
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,128,64,8,0,0.10052266716957092
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,128,64,16,0,0.0631466656923294
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,128,64,32,0,0.04334933559099833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,128,64,4,0,0.17988266547520956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,128,64,64,0,0.02918400118748347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,128,64,2,0,0.3326293428738912
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,128,64,1,0,0.6415359973907471
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,128,128,32,0,0.08055466910203297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,128,128,16,0,0.11502933502197266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,128,128,8,0,0.1955839991569519
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,128,64,1,0,0.6410239934921265
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,64,512,1,0,3.736234664916992
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,128,128,64,0,0.05751466751098633
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,128,128,4,0,0.3490133285522461
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,128,128,16,0,0.11349333326021831
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,128,128,32,0,0.0817493349313736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,128,128,8,0,0.19575466712315878
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,128,128,2,0,0.6609919865926107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,128,128,64,0,0.05717333157857259
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,128,128,4,0,0.35037867228190106
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,128,128,2,0,0.661845326423645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,128,256,16,0,0.22920533021291098
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,128,256,8,0,0.39867734909057617
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,128,256,32,0,0.1508693297704061
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,128,256,64,0,0.10939733187357585
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,128,128,1,0,1.2770986557006836
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,128,256,4,0,0.7359146277109782
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,128,128,1,0,1.2692480087280273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,128,256,8,0,0.398848017056783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,128,256,16,0,0.23688532908757529
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,128,256,32,0,0.15411200126012167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,128,256,64,0,0.11195733149846394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,128,256,2,0,1.407317320505778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,128,256,4,0,0.7369386355082194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,256,16,2,0,0.3911679983139038
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,256,16,4,0,0.2032639980316162
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,256,16,8,0,0.11246933539708455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,256,16,16,0,0.06519466638565063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,256,16,1,0,0.8017919858296713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,256,16,32,0,0.040448000033696495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,256,16,64,0,0.02918400118748347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,256,16,16,0,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,256,16,8,0,0.1114453375339508
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,256,16,4,0,0.2032639980316162
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,256,16,2,0,0.38997332255045575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,256,16,64,0,0.02918400118748347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,256,16,32,0,0.03925333420435587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,256,16,1,0,0.7674880027770996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,128,256,2,0,1.4052693049112956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,256,32,8,0,0.12800000111262003
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,256,32,16,0,0.07645866771539052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,256,32,32,0,0.05205333232879639
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,256,32,4,0,0.2379093368848165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,256,32,64,0,0.03583999971548716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,256,32,2,0,0.4500480095545451
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,256,32,16,0,0.07594666878382365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,256,32,8,0,0.1281706690788269
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,256,32,4,0,0.23825067281723022
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,256,32,32,0,0.051029334465662636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,256,32,1,0,0.8729600111643473
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,256,32,2,0,0.44970667362213135
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,256,32,64,0,0.03549866626660029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,128,256,1,0,2.7450027465820312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,256,64,16,0,0.11571199695269267
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,256,64,32,0,0.0820906658967336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,256,32,1,0,0.876202662785848
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,256,64,8,0,0.18926932414372763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,256,64,64,0,0.057002668579419456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,256,64,4,0,0.3485013246536255
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,256,64,16,0,0.11673600474993388
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,256,64,8,0,0.19336533546447754
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,256,64,4,0,0.34594134489695233
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,256,64,2,0,0.656554659207662
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,256,64,64,0,0.05734399954477946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,256,64,32,0,0.08055466910203297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,128,256,1,0,2.7426134745279946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,256,64,2,0,0.6594560146331787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,256,128,32,0,0.14421332875887552
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,256,128,8,0,0.3781973520914714
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,256,128,16,0,0.22272000710169473
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,256,64,1,0,1.2690773010253906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,1,256,128,64,0,0.13107200463612875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,256,64,1,0,1.2675413290659587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,256,128,4,0,0.688810666402181
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,256,128,32,0,0.14131200313568115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,256,128,8,0,0.39099733034769696
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,256,128,16,0,0.2239146629969279
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,1,16,1,0,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,1,256,128,64,0,0.10598400235176086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,1,16,2,0,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,1,16,4,0,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,256,128,2,0,1.3100372950236003
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,256,128,4,0,0.6913706461588541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,1,16,8,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,1,16,16,0,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,1,16,32,0,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,1,16,64,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,1,16,1,0,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,1,16,2,0,0.01791999985774358
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,1,16,4,0,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,1,16,8,0,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,1,16,16,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,1,16,32,0,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,1,16,64,0,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,1,32,1,0,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,1,32,2,0,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,1,32,4,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,1,32,16,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,1,32,8,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,1,32,32,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,1,32,64,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,1,32,1,0,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,1,32,2,0,0.018432000031073887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,1,32,4,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,1,32,8,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,1,32,16,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,1,32,32,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,1,32,64,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,1,64,4,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,1,64,2,0,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,1,64,1,0,0.020309332758188248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,1,64,8,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,1,64,16,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,256,128,2,0,1.3071359793345134
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,1,64,32,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,1,64,64,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,1,64,1,0,0.020309332758188248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,1,64,2,0,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,1,64,4,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,1,64,8,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,1,64,16,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,1,64,32,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,1,64,64,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,1,128,1,0,0.028160000840822857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,1,128,2,0,0.020992000897725422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,1,128,4,0,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,1,128,8,0,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,1,128,16,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,1,128,32,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,1,128,64,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,1,128,2,0,0.021162666380405426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,1,128,1,0,0.0290133332212766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,1,128,4,0,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,1,128,8,0,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,1,128,16,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,1,128,32,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,1,128,64,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,1,256,1,0,0.05870933334032694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,1,256,2,0,0.03566933423280716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,1,256,8,0,0.022357332209746044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,1,256,4,0,0.027647999425729115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,1,256,16,0,0.020992000897725422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,1,256,32,0,0.020992000897725422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,1,256,64,0,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,1,256,4,0,0.027647999425729115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,1,256,8,0,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,1,256,1,0,0.05922133227189382
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,1,256,2,0,0.0365226666132609
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,1,256,32,0,0.020821332931518555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,1,256,16,0,0.020992000897725422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,256,128,1,0,2.5535146395365396
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,1,256,64,0,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,1,512,16,0,0.034304000437259674
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,1,512,8,0,0.04130133241415024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,1,512,4,0,0.05000533163547516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,1,512,2,0,0.07850666840871175
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,1,512,32,0,0.030037333567937214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,1,512,1,0,0.13090133666992188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,1,512,64,0,0.02935466667016347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,1,512,8,0,0.04113066693147024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,1,512,2,0,0.07816533247629802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,1,512,1,0,0.13038933277130127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,1,512,16,0,0.034645333886146545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,1,512,32,0,0.030207999050617218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,1,512,4,0,0.05120000243186951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,1,512,64,0,0.02918400118748347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,1,1024,16,0,0.06877866884072621
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,1,1024,8,0,0.07338666419188182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,1,1024,32,0,0.053247998158137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,1,1024,64,0,0.04795733094215393
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,1,1024,4,0,0.12339199582735698
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,1,1024,2,0,0.2039466698964437
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,256,128,1,0,2.546346664428711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,1,1024,1,0,0.381440003712972
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,1,1024,16,0,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,1,1024,32,0,0.053247998158137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,1,1024,8,0,0.07543466488520305
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,1,1024,64,0,0.048469334840774536
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,1,1024,4,0,0.1230506698290507
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,1,1024,2,0,0.2039466698964437
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,1,1024,1,0,0.3781973520914714
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,1,1536,16,0,0.09693866968154907
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,1,1536,64,0,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,1,1536,8,0,0.14762666821479797
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,1,1536,32,0,0.08157866696516673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,1,1536,1,0,0.7594666481018066
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,1,1536,4,0,0.22715733448664346
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,1,1536,2,0,0.40174933274586994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,1,1536,4,0,0.22323199113210043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,1,1536,32,0,0.08140799899895985
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,1,1536,64,0,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,1,1536,16,0,0.09693866968154907
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,1,1536,8,0,0.14660267035166422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,1,1536,2,0,0.3997013171513875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,1,1536,1,0,0.7565653324127197
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,1,2048,16,0,0.1384106675783793
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,1,2048,8,0,0.21606399615605673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,1,2048,4,0,0.359935998916626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,1,2048,64,0,0.08311466872692108
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,1,2048,32,0,0.12322133779525757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,1,2048,2,0,0.6536533435185751
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,1,2048,32,0,0.12322133779525757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,1,2048,16,0,0.13772799571355185
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,1,2048,64,0,0.08294400076071422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,1,2048,8,0,0.21691733598709106
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,1,2048,1,0,1.2603733539581299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,1,2048,4,0,0.35942399501800537
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,1,2048,2,0,0.6568959951400757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,1,2048,1,0,1.2615679899851482
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,1,3072,64,0,0.16247466206550598
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,1,3072,32,0,0.1802240014076233
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,1,3072,16,0,0.2409813404083252
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,1,3072,8,0,0.4078933397928874
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,1,3072,4,0,0.7224319775899252
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,1,3072,16,0,0.24780799945195517
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,1,3072,8,0,0.40994131565093994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,1,3072,32,0,0.17971199750900269
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,1,3072,2,0,1.345194657643636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,1,3072,4,0,0.7248213291168213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,1,3072,64,0,0.16247466206550598
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,1,3072,2,0,1.3542399406433105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,1,3072,1,0,2.6280959447224936
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,1,4096,8,0,0.6712319850921631
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,1,4096,16,0,0.3979946772257487
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,1,4096,32,0,0.26658133665720624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,1,4096,64,0,0.18227199713389078
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,1,4096,4,0,1.2136106491088867
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,1,3072,1,0,2.649770736694336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,1,4096,2,0,2.310485363006592
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,1,4096,32,0,0.26999467611312866
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,1,4096,16,0,0.4049919843673706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,1,4096,8,0,0.6604799826939901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,1,4096,4,0,1.2115626335144043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,1,4096,64,0,0.18056533734003702
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,1,4096,2,0,2.3074132601420083
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,1,4096,1,0,4.682751973470052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,1,6144,16,0,0.7830186684926351
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,1,6144,32,0,0.47359999020894367
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,1,6144,8,0,1.373696009318034
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,1,6144,64,0,0.3500373363494873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,1,6144,4,0,2.5606826146443686
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,1,4096,1,0,4.655616124471028
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,1,6144,16,0,0.7772160371144613
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,1,6144,32,0,0.47598934173583984
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,1,6144,8,0,1.394005298614502
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,1,6144,2,0,5.0510508219401045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,1,6144,4,0,2.5775787035624185
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,1,6144,64,0,0.34628268082936603
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,1,8192,16,0,1.2799999713897705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,1,6144,2,0,5.083648045857747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,1,8192,8,0,2.3185067176818848
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,1,8192,32,0,0.7531519730885824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,1,8192,64,0,0.5094399849573771
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,1,8192,4,0,4.4366505940755205
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,1,6144,1,0,10.187605539957682
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,1,6144,1,0,10.297856012980143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,1,8192,16,0,1.2781226634979248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,1,8192,8,0,2.334719975789388
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,1,8192,2,0,8.993450800577799
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,1,8192,64,0,0.509610652923584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,1,8192,32,0,0.7524693012237549
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,1,8192,4,0,4.509183883666992
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,1,10240,16,0,1.9182933171590169
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,1,10240,8,0,3.573418617248535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,1,8192,2,0,8.751103719075521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,1,10240,32,0,1.0810026327768962
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,1,10240,64,0,0.7161173025767008
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,1,10240,4,0,7.065429051717122
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,1,8192,1,0,17.936725616455078
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,1,10240,16,0,1.9131733576456706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,1,10240,8,0,3.5590826670328775
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,1,10240,2,0,13.777236938476562
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,1,10240,4,0,7.028053283691406
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,1,10240,32,0,1.1096746921539307
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,1,10240,64,0,0.7128746509552002
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,1,8192,1,0,18.02837371826172
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,1,12288,16,0,2.668373425801595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,1,12288,8,0,5.113343874613444
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,1,10240,2,0,13.8067626953125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,1,12288,32,0,1.5447039604187012
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,1,12288,64,0,0.9448106288909912
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,1,12288,4,0,10.107391993204752
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,1,10240,1,0,27.389439900716145
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,1,12288,8,0,5.139797210693359
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,1,12288,16,0,2.678442637125651
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,1,12288,64,0,0.9477120240529379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,1,12288,32,0,1.5496533711751301
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,1,12288,2,0,19.876522064208984
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,1,12288,4,0,9.97819709777832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,1,10240,1,0,27.10681660970052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,1,16384,16,0,4.5400746663411455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,1,16384,8,0,8.949759801228842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,1,12288,2,0,19.867989857991535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,1,16384,32,0,2.508117357889811
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,1,16384,64,0,1.4598827362060547
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,1,16384,4,0,17.54043706258138
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,1,12288,1,0,39.61241658528646
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,1,16384,8,0,8.996351877848307
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,1,16384,16,0,4.607829411824544
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,1,16384,32,0,2.51255464553833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,1,16384,4,0,17.7073491414388
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,1,16384,2,0,34.85303497314453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,1,16384,64,0,1.4574933052062988
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,1,12288,1,0,39.15946706136068
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,2,16,2,0,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,2,16,1,0,0.025429333249727886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,2,16,4,0,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,2,16,16,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,2,16,32,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,2,16,8,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,2,16,64,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,2,16,1,0,0.025429333249727886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,2,16,2,0,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,2,16,4,0,0.018432000031073887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,2,16,16,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,2,16,8,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,2,16,32,0,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,2,16,64,0,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,2,32,2,0,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,2,32,1,0,0.02628266563018163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,2,32,16,0,0.015189333508412043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,2,32,8,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,2,32,4,0,0.018432000031073887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,2,32,32,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,2,32,64,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,1,16384,2,0,34.32960001627604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,2,32,1,0,0.025770666698614757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,2,32,2,0,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,2,32,8,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,2,32,16,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,2,32,4,0,0.018602666755517323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,2,32,32,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,2,32,64,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,2,64,4,0,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,2,64,1,0,0.027306665976842243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,2,64,2,0,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,2,64,8,0,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,2,64,16,0,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,2,64,32,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,2,64,64,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,2,64,1,0,0.027306665976842243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,2,64,4,0,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,2,64,2,0,0.02065066620707512
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,2,64,8,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,2,64,16,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,2,64,32,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,2,64,64,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,2,128,2,0,0.028330666323502857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,2,128,8,0,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,2,128,16,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,2,128,1,0,0.0506879985332489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,2,128,32,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,2,128,4,0,0.021162666380405426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,2,128,64,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,2,128,2,0,0.02867199977238973
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,2,128,1,0,0.05120000243186951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,2,128,4,0,0.021162666380405426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,2,128,8,0,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,2,128,16,0,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,2,128,32,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,2,128,64,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,2,256,4,0,0.03618133316437403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,2,256,8,0,0.027818667391935985
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,2,256,2,0,0.05853866537412008
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,2,256,16,0,0.022357332209746044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,2,256,1,0,0.12100266416867574
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,2,256,32,0,0.021162666380405426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,2,256,64,0,0.021674667795499165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,2,256,4,0,0.0363520011305809
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,2,256,8,0,0.027647999425729115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,2,256,16,0,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,2,256,32,0,0.021162666380405426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,2,256,2,0,0.058880001306533813
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,2,256,1,0,0.09830400347709656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,1,16384,1,0,69.21659851074219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,2,256,64,0,0.020992000897725422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,2,512,4,0,0.077824001510938
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,2,512,16,0,0.04164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,2,512,8,0,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,2,512,64,0,0.03054933249950409
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,2,512,32,0,0.034304000437259674
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,2,512,2,0,0.13090133666992188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,2,512,1,0,0.244053324063619
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,2,512,8,0,0.051029334465662636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,2,512,16,0,0.04113066693147024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,2,512,32,0,0.034645333886146545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,2,512,2,0,0.13141333063443503
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,2,512,64,0,0.03054933249950409
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,2,512,4,0,0.07850666840871175
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,2,512,1,0,0.24337067206700644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,2,1024,16,0,0.07867733140786488
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,2,1024,32,0,0.06877866884072621
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,2,1024,64,0,0.053930665055910744
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,1,16384,1,0,70.88742574055989
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,2,1024,8,0,0.12270933389663696
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,2,1024,4,0,0.20684800545374551
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,2,1024,2,0,0.3925333420435588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,2,1024,1,0,0.7316479682922363
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,2,1024,32,0,0.06877866884072621
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,2,1024,16,0,0.07645866771539052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,2,1024,4,0,0.20718934138615927
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,2,1024,64,0,0.053930665055910744
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,2,1024,8,0,0.12390399972597758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,2,1024,2,0,0.38280534744262695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,2,1536,16,0,0.1430186629295349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,2,1536,32,0,0.0981333355108897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,2,1536,8,0,0.22835199038187662
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,2,1024,1,0,0.7321600119272867
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,2,1536,64,0,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,2,1536,4,0,0.3991893529891968
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,2,1536,2,0,0.7541759808858236
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,2,1536,32,0,0.09796266754468282
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,2,1536,8,0,0.226474662621816
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,2,1536,16,0,0.14404267072677612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,2,1536,4,0,0.4041386842727661
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,2,1536,64,0,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,2,1536,2,0,0.7533226807912191
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,2,1536,1,0,1.4682453473409016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,2,2048,8,0,0.36317865053812665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,2,2048,16,0,0.22050132354100546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,2,2048,32,0,0.13943466544151306
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,2,2048,64,0,0.12526933352152506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,2,2048,4,0,0.6574079990386963
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,2,1536,1,0,1.4673919677734375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,2,2048,2,0,1.252522627512614
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,2,2048,16,0,0.21572266022364298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,2,2048,32,0,0.13414399822553
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,2,2048,8,0,0.3604480028152466
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,2,2048,4,0,0.6584320068359375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,2,2048,64,0,0.1250986655553182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,2,2048,2,0,1.2571307023366292
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,2,3072,16,0,0.41557331879933673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,2,2048,1,0,2.4506026903788247
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,2,3072,8,0,0.7224319775899252
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,2,3072,32,0,0.24883200724919638
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,2,3072,4,0,1.358847935994466
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,2,3072,64,0,0.18363734086354574
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,2,3072,16,0,0.41557331879933673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,2,2048,1,0,2.4751787185668945
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,2,3072,8,0,0.7338666915893555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,2,3072,2,0,2.6175146102905273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,2,3072,4,0,1.3607254028320312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,2,3072,64,0,0.1843199928601583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,2,3072,32,0,0.24320000410079956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,2,3072,2,0,2.6231466929117837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,2,4096,16,0,0.6732800006866455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,2,4096,8,0,1.222314675649007
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,2,4096,32,0,0.4077226718266805
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,2,4096,64,0,0.27187200387318927
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,2,4096,4,0,2.3135573069254556
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,2,3072,1,0,5.3886292775472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,2,4096,16,0,0.67413330078125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,2,3072,1,0,5.2616532643636065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,2,4096,2,0,4.584959983825684
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,2,4096,8,0,1.2224853038787842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,2,4096,64,0,0.27323732773462933
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,2,4096,32,0,0.4154026508331299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,2,4096,4,0,2.315605322519938
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,2,4096,2,0,4.680874824523926
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,2,6144,16,0,1.3803520202636719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,2,6144,8,0,2.59549872080485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,2,6144,64,0,0.488106648127238
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,2,6144,32,0,0.7944533030192057
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,2,4096,1,0,9.387349446614584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,2,6144,4,0,5.003605206807454
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,2,4096,1,0,9.364138921101889
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,2,6144,16,0,1.3757440249125164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,2,6144,8,0,2.5859413146972656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,2,6144,2,0,10.140501022338867
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,2,6144,32,0,0.7821653683980306
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,2,6144,64,0,0.4828159809112549
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,2,6144,4,0,5.139626820882161
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,2,8192,16,0,2.3488853772481284
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,2,6144,2,0,10.13043212890625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,2,8192,8,0,4.4446719487508135
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,2,8192,32,0,1.299455960591634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,2,8192,64,0,0.7620266278584799
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,2,8192,4,0,9.072469075520834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,2,6144,1,0,20.009131113688152
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,2,8192,8,0,4.512938817342122
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,2,8192,16,0,2.3490559260050454
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,2,8192,2,0,17.799339294433594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,2,6144,1,0,20.44808578491211
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,2,8192,4,0,8.87381362915039
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,2,8192,32,0,1.2929706573486328
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,2,8192,64,0,0.7630506356557211
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,2,10240,16,0,3.54201602935791
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,2,8192,2,0,17.903274536132812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,2,10240,8,0,7.076864242553711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,2,10240,32,0,1.9268266359965007
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,2,10240,64,0,1.1050666968027751
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,2,10240,4,0,13.788501739501953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,2,8192,1,0,35.69868723551432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,2,10240,8,0,6.981290817260742
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,2,10240,4,0,13.945173899332682
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,2,10240,16,0,3.6031147638956704
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,2,10240,2,0,27.582293192545574
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,2,10240,32,0,1.9406506220499675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,2,10240,64,0,1.1205973625183105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,2,8192,1,0,35.32868194580078
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,2,10240,2,0,27.24744415283203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,2,12288,16,0,4.997632026672363
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,2,12288,8,0,10.021546681722006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,2,12288,64,0,1.5527253150939941
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,2,12288,32,0,2.702335993448893
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,2,12288,4,0,19.601236979166668
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,2,10240,1,0,55.48953755696615
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,2,12288,8,0,9.920170466105143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,2,12288,16,0,5.110613187154134
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,2,12288,32,0,2.7040427525838218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,2,12288,4,0,19.628543853759766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,2,12288,2,0,39.404884338378906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,2,12288,64,0,1.5708160400390625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,2,10240,1,0,55.1178232828776
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,2,16384,16,0,9.031509399414062
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,2,12288,2,0,38.91421763102213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,2,16384,8,0,17.68567403157552
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,2,16384,32,0,4.674730618794759
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,2,16384,64,0,2.5335466066996255
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,2,16384,4,0,34.90013885498047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,2,12288,1,0,80.21708679199219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,2,16384,8,0,17.498111724853516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,2,16384,16,0,8.935424168904623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,2,12288,1,0,79.87951151529948
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,2,16384,4,0,34.143402099609375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,2,16384,32,0,4.661418596903483
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,2,16384,64,0,2.540544033050537
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,4,16,1,0,0.034815999368826546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,4,16,2,0,0.025087999800841015
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,2,16384,2,0,69.15191650390625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,4,16,4,0,0.01877333347996076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,4,16,8,0,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,4,16,32,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,4,16,16,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,4,16,64,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,4,16,1,0,0.034645333886146545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,4,16,2,0,0.02491733431816101
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,4,16,4,0,0.01877333347996076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,4,16,8,0,0.01791999985774358
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,4,16,16,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,4,16,32,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,4,16,64,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,4,32,1,0,0.03601066768169403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,4,32,2,0,0.02611200014750163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,4,32,4,0,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,4,32,8,0,0.01877333347996076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,4,32,16,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,4,32,32,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,4,32,2,0,0.02611200014750163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,4,32,64,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,4,32,1,0,0.03601066768169403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,4,32,4,0,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,4,32,8,0,0.018602666755517323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,4,32,16,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,4,32,32,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,4,32,64,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,4,64,2,0,0.027306665976842243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,4,64,1,0,0.05034666756788889
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,4,64,8,0,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,4,64,4,0,0.02065066620707512
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,4,64,16,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,4,64,32,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,4,64,64,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,4,64,4,0,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,4,64,2,0,0.027306665976842243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,4,64,1,0,0.05000533163547516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,4,64,8,0,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,4,64,16,0,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,4,64,32,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,4,64,64,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,4,128,4,0,0.028501334289709728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,4,128,2,0,0.05120000243186951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,4,128,1,0,0.09215999643007915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,4,128,8,0,0.021162666380405426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,4,128,16,0,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,4,128,32,0,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,4,128,64,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,2,16384,2,0,70.17284138997395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,4,128,4,0,0.028501334289709728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,4,128,2,0,0.0506879985332489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,4,128,1,0,0.09028266867001851
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,4,128,8,0,0.020992000897725422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,4,128,16,0,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,4,128,32,0,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,4,128,64,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,4,256,8,0,0.036864000062147774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,4,256,16,0,0.028160000840822857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,4,256,4,0,0.05905066430568695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,4,256,32,0,0.022357332209746044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,4,256,2,0,0.09779199957847595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,4,256,1,0,0.18227199713389078
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,4,256,64,0,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,4,256,8,0,0.03703466554482778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,4,256,4,0,0.059392000238100685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,4,256,16,0,0.027989332874615986
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,4,256,2,0,0.09745066364606221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,4,256,32,0,0.02218666672706604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,4,256,64,0,0.021674667795499165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,4,256,1,0,0.18261333306630453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,4,512,8,0,0.07935999830563863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,4,512,4,0,0.13141333063443503
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,4,512,16,0,0.05239466826121012
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,4,512,64,0,0.03532800078392029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,4,512,32,0,0.04147200038035711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,4,512,2,0,0.24593067169189453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,4,512,1,0,0.47274665037790936
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,4,512,16,0,0.05256533126036326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,4,512,8,0,0.07867733140786488
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,4,512,4,0,0.130730668703715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,4,512,2,0,0.24627200762430826
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,4,512,32,0,0.04181333382924398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,4,512,64,0,0.03532800078392029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,4,512,1,0,0.46967466672261554
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,4,1024,16,0,0.12663466731707254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,4,1024,8,0,0.2121386726697286
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,4,1024,32,0,0.077824001510938
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,4,1024,4,0,0.38331735134124756
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,4,1024,64,0,0.06980266670385997
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,4,1024,2,0,0.7325013478597006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,4,1024,4,0,0.38331735134124756
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,4,1024,8,0,0.21094399690628052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,4,1024,16,0,0.1269760032494863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,4,1024,64,0,0.06997333467006683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,4,1024,32,0,0.07850666840871175
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,4,1024,2,0,0.7304533322652181
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,4,1024,1,0,1.433087984720866
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,4,1536,16,0,0.2302293380101522
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,2,16384,1,0,138.77691650390625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,4,1536,8,0,0.4113066593805949
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,4,1024,1,0,1.437013308207194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,4,1536,32,0,0.14643200238545737
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,4,1536,4,0,0.7586133480072021
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,4,1536,64,0,0.10410666465759277
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,4,1536,8,0,0.40567465623219806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,4,1536,16,0,0.233130673567454
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,4,1536,2,0,1.4661973317464192
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,4,1536,4,0,0.7616853713989258
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,4,1536,32,0,0.14523733655611673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,4,1536,64,0,0.10308266679445903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,4,1536,2,0,1.473365306854248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,4,2048,8,0,0.6715733210245768
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,4,1536,1,0,2.9129387537638345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,4,2048,16,0,0.3684693177541097
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,4,2048,32,0,0.22272000710169473
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,4,2048,64,0,0.1469439963499705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,4,2048,4,0,1.2608853181203206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,4,1536,1,0,2.919935862223307
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,4,2048,2,0,2.458965301513672
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,4,2048,8,0,0.6652586857477824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,2,16384,1,0,140.42129516601562
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,4,2048,16,0,0.3689813216527303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,4,2048,32,0,0.22357332706451416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,4,2048,4,0,1.2583253383636475
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,4,2048,64,0,0.14506666858990988
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,4,3072,16,0,0.7328426837921143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,4,3072,8,0,1.3721599578857422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,4,2048,2,0,2.4574294090270996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,4,2048,1,0,4.978858629862468
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,4,3072,32,0,0.4215466578801473
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,4,3072,64,0,0.2524159948031108
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,4,3072,4,0,2.6374826431274414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,4,3072,16,0,0.7372799714406332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,4,3072,8,0,1.374549388885498
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,4,2048,1,0,5.0664107004801435
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,4,3072,32,0,0.4256426493326823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,4,3072,2,0,5.393920262654622
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,4,3072,4,0,2.6431147257486978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,4,3072,64,0,0.25890133778254193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,4,4096,16,0,1.2311893304189045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,4,4096,8,0,2.337792078653971
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,4,3072,2,0,5.333162625630696
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,4,4096,32,0,0.6842026710510254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,4,4096,64,0,0.41625599066416424
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,4,4096,4,0,4.679338773091634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,4,3072,1,0,10.878805796305338
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,4,4096,16,0,1.2262399991353352
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,4,4096,8,0,2.3309653600056968
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,4,4096,64,0,0.41813333829243976
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,4,4096,32,0,0.6917119820912679
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,4,4096,4,0,4.671317418416341
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,4,3072,1,0,10.734079996744791
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,4,4096,2,0,9.349632263183594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,4,6144,16,0,2.604544003804525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,4,4096,2,0,9.126911799112955
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,4,6144,8,0,5.105322519938151
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,4,6144,64,0,0.8132266998291016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,4,6144,32,0,1.4057812690734863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,4,6144,4,0,10.165760040283203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,4,4096,1,0,18.72742462158203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,4,6144,16,0,2.613248030344645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,4,6144,8,0,5.157034556070964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,4,4096,1,0,18.43404769897461
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,4,6144,4,0,10.257919947306315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,4,6144,32,0,1.412266731262207
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,4,6144,64,0,0.8132266998291016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,4,6144,2,0,20.42845916748047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,4,8192,16,0,4.542122522989909
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,4,8192,8,0,8.93405850728353
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,4,8192,32,0,2.354858716328939
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,4,6144,2,0,20.001280466715496
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,4,8192,64,0,1.3315412998199463
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,4,8192,4,0,17.921707153320312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,4,6144,1,0,41.10284678141276
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,4,8192,8,0,8.997546513875326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,4,8192,32,0,2.3802879651387534
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,4,8192,4,0,17.881429036458332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,4,8192,16,0,4.568575859069824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,8,16,1,0,0.056832000613212585
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,4,8192,64,0,1.3289813200632732
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,8,16,2,0,0.03515733281771342
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,8,16,4,0,0.025258667767047882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,4,8192,2,0,35.47528584798177
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,8,16,8,0,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,8,16,16,0,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,8,16,32,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,8,16,64,0,0.015360000232855478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,4,6144,1,0,40.1972910563151
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,8,16,2,0,0.034815999368826546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,8,16,1,0,0.056661332647005715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,8,16,4,0,0.025429333249727886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,8,16,16,0,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,8,16,8,0,0.01877333347996076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,8,16,32,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,8,16,64,0,0.015530666957298914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,8,32,1,0,0.06348800162474315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,8,32,4,0,0.025770666698614757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,8,32,2,0,0.03583999971548716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,8,32,8,0,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,8,32,16,0,0.018432000031073887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,8,32,32,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,8,32,64,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,8,32,2,0,0.03618133316437403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,8,32,1,0,0.06348800162474315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,8,32,8,0,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,8,32,16,0,0.018602666755517323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,8,32,4,0,0.02628266563018163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,8,32,32,0,0.016042667130629223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,8,32,64,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,8,64,4,0,0.027818667391935985
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,8,64,2,0,0.05034666756788889
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,8,64,1,0,0.09011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,8,64,8,0,0.02065066620707512
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,8,64,16,0,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,8,64,32,0,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,8,64,64,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,8,64,4,0,0.027647999425729115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,8,64,2,0,0.05017599960168203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,8,64,8,0,0.02065066620707512
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,8,64,16,0,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,8,64,1,0,0.0890880028406779
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,4,8192,2,0,35.07677968343099
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,8,64,32,0,0.01621333385507266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,8,64,64,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,8,128,4,0,0.05120000243186951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,8,128,8,0,0.029525332152843475
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,8,128,16,0,0.021674667795499165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,8,128,32,0,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,8,128,2,0,0.09079466263453166
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,8,128,1,0,0.16964266697565714
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,8,128,64,0,0.01672533278663953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,8,128,8,0,0.030378667016824085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,8,128,4,0,0.05120000243186951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,8,128,16,0,0.021674667795499165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,8,128,2,0,0.09113599856694539
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,8,128,32,0,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,8,128,64,0,0.01672533278663953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,8,128,1,0,0.17083734273910522
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,8,256,8,0,0.0602453351020813
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,8,256,4,0,0.09915733337402344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,8,256,16,0,0.03822933385769526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,8,256,32,0,0.028501334289709728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,8,256,2,0,0.18193066120147705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,8,256,64,0,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,8,256,1,0,0.3510613441467285
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,8,256,16,0,0.03857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,8,256,8,0,0.060415998101234436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,8,256,4,0,0.09898666540781657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,8,256,2,0,0.18261333306630453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,8,256,32,0,0.028501334289709728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,8,256,64,0,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,8,256,1,0,0.35072000821431476
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,8,512,16,0,0.08140799899895985
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,8,512,8,0,0.1360213359196981
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,8,512,4,0,0.24832000335057577
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,8,512,32,0,0.054272000988324486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,8,512,64,0,0.043178667624791466
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,8,512,2,0,0.4724053144454956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,8,512,8,0,0.13636266191800436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,8,512,4,0,0.24729599555333456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,8,512,16,0,0.08140799899895985
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,8,512,32,0,0.054272000988324486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,8,512,1,0,0.928938627243042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,8,512,2,0,0.47223468621571857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,8,512,64,0,0.04334933559099833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,4,8192,1,0,70.96507771809895
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,8,1024,16,0,0.21606399615605673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,8,512,1,0,0.9263786474863688
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,8,1024,8,0,0.3942399819691976
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,8,1024,64,0,0.08516266942024231
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,8,1024,32,0,0.13243732849756876
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,8,1024,4,0,0.7432533105214437
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,8,1024,16,0,0.2172586719195048
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,8,1024,2,0,1.4334294001261394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,8,1024,8,0,0.39321601390838623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,8,1024,32,0,0.13260799646377563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,8,1024,4,0,0.7420586744944254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,8,1024,64,0,0.08430932958920796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,8,1024,2,0,1.4341120719909668
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,8,1536,8,0,0.7743146419525146
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,8,1536,16,0,0.4150613149007161
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,8,1024,1,0,2.8357973098754883
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,8,1536,32,0,0.24081067244211832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,8,1536,4,0,1.4776320457458496
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,8,1536,64,0,0.15684266885121664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,4,8192,1,0,72.30959065755208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,8,1536,16,0,0.4198400179545085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,8,1024,1,0,2.887338638305664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,8,1536,8,0,0.7736319700876871
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,8,1536,2,0,2.921130816141764
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,8,1536,64,0,0.1532586713631948
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,8,1536,32,0,0.24337067206700644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,8,1536,4,0,1.4771199226379395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,8,2048,16,0,0.6768639882405599
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,8,2048,8,0,1.2803413073221843
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,8,1536,2,0,2.926080067952474
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,8,2048,32,0,0.3829760154088338
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,8,2048,64,0,0.23193599780400595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,8,2048,4,0,2.47432533899943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,8,1536,1,0,5.958656311035156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,8,2048,16,0,0.6770346959431967
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,8,2048,8,0,1.275391976038615
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,8,2048,64,0,0.23466666539510092
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,8,2048,4,0,2.4683519999186196
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,8,2048,32,0,0.3804159959157308
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,8,2048,2,0,5.064703941345215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,8,1536,1,0,5.902336120605469
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,8,2048,2,0,5.006848017374675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,8,3072,16,0,1.4032212893168132
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,8,3072,8,0,2.681002616882324
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,8,3072,64,0,0.44151465098063153
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,8,3072,32,0,0.7620266278584799
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,8,2048,1,0,10.24733861287435
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,8,3072,4,0,5.352959950764974
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,8,3072,16,0,1.3818880716959636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,8,2048,1,0,10.24887466430664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,8,3072,8,0,2.6586453119913735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,8,3072,32,0,0.7507627010345459
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,8,3072,4,0,5.372245152791341
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,8,3072,64,0,0.44339199860890705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,8,3072,2,0,10.782549540201822
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,8,4096,16,0,2.369194666544596
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,8,4096,8,0,4.695210774739583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,8,4096,32,0,1.2554240226745605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,8,3072,2,0,10.579968134562174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,8,4096,64,0,0.711680014928182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,8,4096,4,0,9.262592315673828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,8,3072,1,0,21.772288004557293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,8,4096,8,0,4.65937074025472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,8,4096,4,0,9.36516253153483
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,8,4096,32,0,1.26310396194458
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,8,4096,16,0,2.357418696085612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,16,16,1,0,0.1032533347606659
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,16,16,2,0,0.05717333157857259
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,8,4096,64,0,0.7130453586578369
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,16,16,4,0,0.034815999368826546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,8,4096,2,0,18.36629358927409
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,8,3072,1,0,21.59888966878255
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,16,16,8,0,0.025087999800841015
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,16,16,16,0,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,16,16,32,0,0.018432000031073887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,16,16,64,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,16,16,4,0,0.034815999368826546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,16,16,2,0,0.056661332647005715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,16,16,1,0,0.10376532872517903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,16,16,8,0,0.025087999800841015
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,16,16,16,0,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,16,16,32,0,0.01826133330663045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,16,16,64,0,0.01570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,16,32,4,0,0.03601066768169403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,16,32,2,0,0.06331733365853627
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,16,32,8,0,0.02611200014750163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,16,32,1,0,0.11315199732780457
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,16,32,16,0,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,16,32,32,0,0.018602666755517323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,16,32,64,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,16,32,2,0,0.06382933259010315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,16,32,1,0,0.11400533715883891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,16,32,16,0,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,16,32,8,0,0.02611200014750163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,16,32,4,0,0.0363520011305809
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,16,32,32,0,0.01877333347996076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,16,32,64,0,0.015872000406185787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,16,64,8,0,0.028160000840822857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,16,64,16,0,0.020992000897725422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,16,64,2,0,0.08806399504343669
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,16,64,4,0,0.05085866649945577
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,16,64,1,0,0.16674133141835532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,16,64,32,0,0.019797333826621372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,16,64,64,0,0.016384000579516094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,16,64,4,0,0.05085866649945577
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,16,64,16,0,0.021162666380405426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,16,64,8,0,0.028501334289709728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,16,64,32,0,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,16,64,64,0,0.016554666062196095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,16,64,1,0,0.16742400328318277
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,8,4096,2,0,18.20910898844401
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,16,64,2,0,0.08840533097585042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,16,128,16,0,0.032255999743938446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,16,128,32,0,0.02218666672706604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,16,128,8,0,0.05273599922657013
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,16,128,4,0,0.09113599856694539
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,16,128,2,0,0.17100799083709717
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,16,128,64,0,0.021162666380405426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,16,128,8,0,0.052906667192777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,16,128,4,0,0.09250133236249287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,16,128,16,0,0.0315733328461647
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,16,128,1,0,0.32716800769170123
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,16,128,2,0,0.16964266697565714
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,16,128,32,0,0.02218666672706604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,16,128,64,0,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,16,128,1,0,0.3269973397254944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,16,256,16,0,0.06400000055631001
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,16,256,8,0,0.10291199882825215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,16,256,4,0,0.18602667252222696
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,16,256,32,0,0.04130133241415024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,16,256,64,0,0.02935466667016347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,16,256,2,0,0.3519146839777629
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,16,256,8,0,0.10291199882825215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,16,256,16,0,0.06485333542029063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,16,256,4,0,0.18602667252222696
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,16,256,32,0,0.04130133241415024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,16,256,64,0,0.029696000119050343
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,16,256,2,0,0.35174401601155597
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,16,256,1,0,0.6910293102264404
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,16,512,16,0,0.14335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,16,512,8,0,0.2571946581204732
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,16,256,1,0,0.6929066975911459
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,16,512,32,0,0.08755200107892354
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,16,512,4,0,0.48213334878285724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,16,512,64,0,0.06007466713587443
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,8,4096,1,0,37.45791880289713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,16,512,8,0,0.25890133778254193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,16,512,2,0,0.9282560348510742
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,16,512,16,0,0.1423360009988149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,16,512,64,0,0.06178133189678192
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,16,512,4,0,0.4814506769180298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,16,512,32,0,0.08635733524958293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,16,512,2,0,0.9314986864725748
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,16,1024,8,0,0.7569066683451334
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,16,512,1,0,1.8394452730814617
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,16,1024,16,0,0.40226133664449054
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,16,1024,64,0,0.1443839967250824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,16,1024,32,0,0.23176532983779907
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,8,4096,1,0,37.393750508626304
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,16,1024,4,0,1.4581759770711262
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,16,512,1,0,1.836714744567871
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,16,1024,32,0,0.2336426575978597
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,16,1024,16,0,0.40533332029978436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,16,1024,8,0,0.7540053526560465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,16,1024,64,0,0.1437013347943624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,16,1024,2,0,2.8774401346842446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,16,1024,4,0,1.4479360580444336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,16,1536,16,0,0.7915519873301188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,16,1536,8,0,1.5063039461771648
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,16,1536,32,0,0.4317866563796997
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,16,1536,64,0,0.2568533420562744
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,16,1024,2,0,2.882218678792318
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,16,1536,4,0,2.9453652699788413
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,16,1024,1,0,5.906944274902344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,16,1536,16,0,0.7881386280059814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,16,1536,8,0,1.4981120427449544
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,16,1536,2,0,6.006783803304036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,16,1536,32,0,0.4355413516362508
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,16,1536,64,0,0.2595840096473694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,16,1024,1,0,5.7642669677734375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,16,1536,4,0,2.9550933837890625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,16,2048,16,0,1.2950186729431152
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,16,1536,2,0,6.065664291381836
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,16,2048,8,0,2.496682643890381
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,16,2048,64,0,0.408405343691508
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,16,2048,32,0,0.6997333367665609
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,16,2048,4,0,5.0449066162109375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,16,1536,1,0,12.067328135172525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,16,2048,8,0,2.5132373174031577
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,16,2048,16,0,1.307477315266927
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,16,2048,32,0,0.706559975941976
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,16,2048,4,0,5.090304056803386
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,16,2048,64,0,0.40140799681345624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,32,16,1,0,0.19421867529551187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,32,16,4,0,0.0580266664425532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,32,16,2,0,0.10410666465759277
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,16,2048,2,0,10.250069300333658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,32,16,8,0,0.03549866626660029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,16,1536,1,0,11.913387298583984
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,32,16,16,0,0.025941332181294758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,32,16,64,0,0.018602666755517323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,32,16,32,0,0.01911466692884763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,32,16,1,0,0.19353600343068442
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,32,16,4,0,0.05717333157857259
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,32,16,2,0,0.10410666465759277
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,32,16,8,0,0.03515733281771342
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,32,16,16,0,0.025770666698614757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,32,16,32,0,0.01877333347996076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,32,16,64,0,0.01877333347996076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,32,32,4,0,0.06434133152167003
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,32,32,8,0,0.03737599899371465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,32,32,2,0,0.11383466919263203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,32,32,16,0,0.026965332527955372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,32,32,1,0,0.21896533171335855
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,32,32,64,0,0.018944000204404194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,32,32,32,0,0.019626667102177937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,32,32,2,0,0.11332266529401143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,32,32,4,0,0.06434133152167003
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,32,32,8,0,0.036864000062147774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,32,32,16,0,0.02679466704527537
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,32,32,1,0,0.2206719915072123
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,32,32,32,0,0.019968000551064808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,32,32,64,0,0.019285333653291065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,32,64,4,0,0.08994133273760478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,32,64,8,0,0.05273599922657013
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,32,64,16,0,0.02935466667016347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,32,64,2,0,0.1646933356920878
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,32,64,32,0,0.021674667795499165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,32,64,64,0,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,32,64,1,0,0.32153600454330444
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,16,2048,2,0,10.228053410847982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,32,64,8,0,0.05171200136343638
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,32,64,4,0,0.09062400460243225
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,32,64,16,0,0.029696000119050343
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,32,64,32,0,0.021674667795499165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,32,64,2,0,0.1641813317934672
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,32,64,64,0,0.020309332758188248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,32,64,1,0,0.32255999247233075
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,32,128,16,0,0.05614933371543884
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,32,128,32,0,0.036864000062147774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,32,128,8,0,0.0942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,32,128,4,0,0.17254400253295898
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,32,128,64,0,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,32,128,2,0,0.3259733319282532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,32,128,8,0,0.09642666578292847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,32,128,4,0,0.17425066232681274
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,32,128,16,0,0.05580799778302511
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,32,128,32,0,0.0363520011305809
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,32,128,64,0,0.02372266600529353
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,32,128,1,0,0.6360746622085571
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,32,128,2,0,0.32358400026957196
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,32,256,8,0,0.19165867567062378
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,32,256,16,0,0.10871466994285583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,32,128,1,0,0.6389760176340739
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,32,256,32,0,0.06929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,32,256,4,0,0.36027733484903973
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,16,2048,1,0,20.338517506917317
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,32,256,64,0,0.04949333270390829
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,32,256,2,0,0.6920533180236816
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,32,256,16,0,0.10854400197664897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,32,256,8,0,0.19404800732930502
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,32,256,32,0,0.06963199873765309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,32,256,4,0,0.35891199111938477
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,32,256,64,0,0.04863999783992767
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,32,256,2,0,0.6908586819966634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,32,256,1,0,1.366869290669759
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,32,512,8,0,0.49186134338378906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,32,512,32,0,0.1570133368174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,32,512,16,0,0.2658986647923787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,32,512,4,0,0.9442986647288004
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,32,512,64,0,0.09847467144330342
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,32,256,1,0,1.3646507263183594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,16,2048,1,0,19.961514790852863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,32,512,32,0,0.15752533078193665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,32,512,16,0,0.2691413362820943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,32,512,8,0,0.4910080035527547
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,32,512,64,0,0.09915733337402344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,32,512,2,0,1.8461012840270996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,32,512,4,0,0.9432746569315592
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,32,1024,16,0,0.7789226373036703
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,32,512,2,0,1.8430293401082356
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,32,1024,8,0,1.4841173489888508
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,32,1024,64,0,0.25361067056655884
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,32,1024,32,0,0.42769066492716473
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,32,512,1,0,3.744256019592285
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,32,1024,4,0,2.8858025868733725
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,32,1024,16,0,0.7753386497497559
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,32,512,1,0,3.7032960255940757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,32,1024,8,0,1.4685866038004558
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,32,1024,64,0,0.2573653260866801
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,32,1024,32,0,0.425983985265096
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,64,16,1,0,0.3797333240509033
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,64,16,2,0,0.19524266322453818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,64,16,4,0,0.10513066252072652
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,64,16,8,0,0.05819733440876007
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,64,16,16,0,0.03566933423280716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,32,1024,4,0,2.874709447224935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,64,16,32,0,0.0264533335963885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,64,16,64,0,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,32,1024,2,0,5.967018763224284
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,64,16,8,0,0.057855998476346336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,64,16,4,0,0.10564266641934712
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,64,16,2,0,0.1955839991569519
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,64,16,1,0,0.37956265608469647
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,64,16,32,0,0.02611200014750163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,64,16,64,0,0.0194560003777345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,64,16,16,0,0.03566933423280716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,64,32,8,0,0.06587733328342438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,64,32,16,0,0.03857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,64,32,4,0,0.11622400085131328
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,64,32,2,0,0.2198186715443929
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,64,32,32,0,0.027477333943049114
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,64,32,64,0,0.020309332758188248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,64,32,1,0,0.43622398376464844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,64,32,16,0,0.03822933385769526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,64,32,8,0,0.06553600231806438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,64,32,4,0,0.11690666278203328
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,64,32,2,0,0.21947733561197916
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,64,32,32,0,0.027647999425729115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,64,32,1,0,0.4355413516362508
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,64,32,64,0,0.020309332758188248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,64,64,16,0,0.054272000988324486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,64,64,8,0,0.09233066439628601
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,64,64,4,0,0.1723733345667521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,32,1024,2,0,5.97111447652181
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,64,64,64,0,0.022869333624839783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,64,64,2,0,0.3256319959958394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,64,64,32,0,0.03362133353948593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,64,64,16,0,0.054101333022117615
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,64,64,8,0,0.09301333626111348
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,64,64,4,0,0.17203199863433838
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,64,64,1,0,0.6343679825464884
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,64,64,32,0,0.0341333324710528
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,64,64,64,0,0.023039999107519787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,64,64,2,0,0.32614399989446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,64,128,16,0,0.09983999530474345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,64,128,8,0,0.18261333306630453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,64,128,4,0,0.3283626635869344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,64,64,1,0,0.6301013231277466
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,64,128,32,0,0.06263466676076253
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,64,128,64,0,0.04471466441949209
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,64,128,2,0,0.6440960168838501
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,64,128,8,0,0.1807360053062439
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,64,128,4,0,0.32972800731658936
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,64,128,32,0,0.06400000055631001
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,64,128,16,0,0.10291199882825215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,64,128,64,0,0.044031997521718345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,64,128,2,0,0.6435840129852295
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,64,128,1,0,1.2661759853363037
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,32,1024,1,0,11.688106536865234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,64,256,32,0,0.1250986655553182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,64,256,16,0,0.20616533358891806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,64,256,8,0,0.39816534519195557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,64,256,64,0,0.08430932958920796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,64,256,4,0,0.7000746726989746
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,64,128,1,0,1.2673707008361816
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,64,256,16,0,0.20753065745035806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,64,256,8,0,0.3701759974161784
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,64,256,32,0,0.1327786644299825
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,64,256,64,0,0.08311466872692108
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,64,256,2,0,1.3704533576965332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,64,256,4,0,0.7072426478068033
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,32,1024,1,0,11.84000015258789
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,64,512,16,0,0.5195093154907227
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,64,256,2,0,1.3677226702372234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,64,512,64,0,0.18210132916768393
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,64,512,8,0,0.9739946524302164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,64,512,32,0,0.29047467311223346
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,64,256,1,0,2.7185494105021157
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,64,512,4,0,1.8776747385660808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,64,256,1,0,2.7436374028523765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,64,512,16,0,0.5198506514231364
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,64,512,8,0,0.9678506851196289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,64,512,64,0,0.17834667364756265
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,64,512,32,0,0.2943999965985616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,64,512,4,0,1.8804052670796711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,128,16,4,0,0.19831466674804688
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,128,16,2,0,0.38280534744262695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,128,16,1,0,0.7577599684397379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,128,16,8,0,0.10820266604423523
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,128,16,16,0,0.0602453351020813
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,128,16,32,0,0.03703466554482778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,128,16,64,0,0.027136000494162243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,64,512,2,0,3.744938532511393
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,128,16,16,0,0.060415998101234436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,128,16,8,0,0.10734933614730835
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,128,16,4,0,0.19831466674804688
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,128,16,2,0,0.38331735134124756
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,128,16,32,0,0.03703466554482778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,128,16,64,0,0.027136000494162243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,128,16,1,0,0.7616853713989258
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,128,32,16,0,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,128,32,8,0,0.15103999773661295
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,128,32,4,0,0.22818134228388467
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,128,32,2,0,0.43622398376464844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,128,32,32,0,0.04232533276081085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,128,32,64,0,0.0290133332212766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,128,32,8,0,0.12032000223795573
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,128,32,4,0,0.2285226583480835
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,128,32,1,0,0.8686933517456055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,128,32,16,0,0.06877866884072621
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,128,32,32,0,0.04215466479460398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,128,32,2,0,0.4365653196970622
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,128,32,64,0,0.029525332152843475
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,64,512,2,0,3.6978346506754556
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,128,32,1,0,0.8674986362457275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,128,64,16,0,0.0993280013402303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,128,64,8,0,0.17749333381652832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,128,64,32,0,0.06468266745408376
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,128,64,64,0,0.04232533276081085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,128,64,4,0,0.3278506596883138
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,128,64,8,0,0.176639993985494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,128,64,2,0,0.6405119895935059
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,128,64,4,0,0.33450667063395184
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,128,64,32,0,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,128,64,16,0,0.09898666540781657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,128,64,64,0,0.04369066655635834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,128,64,2,0,0.6408533255259196
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,128,64,1,0,1.2590080102284749
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,128,128,8,0,0.34986666838328045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,128,128,16,0,0.19490132729212442
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,128,64,1,0,1.2579840024312336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,128,128,4,0,0.6557013193766276
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,128,128,32,0,0.11588266491889954
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,64,512,1,0,7.678976058959961
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,128,128,64,0,0.08004266520341237
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,128,128,16,0,0.19063466787338257
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,128,128,8,0,0.3510613441467285
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,128,128,32,0,0.11844266454378764
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,128,128,2,0,1.2801706790924072
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,128,128,64,0,0.07987200220425923
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,128,128,4,0,0.6601386864980062
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,64,512,1,0,7.695018768310547
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,128,128,2,0,1.2781226634979248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,128,256,16,0,0.40004265308380127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,128,256,8,0,0.7376213073730469
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,128,256,64,0,0.15121066570281982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,128,256,32,0,0.23654399315516153
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,128,128,1,0,2.5301334063212075
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,128,256,4,0,1.4057812690734863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,128,256,16,0,0.3998719851175944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,128,128,1,0,2.531328042348226
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,128,256,8,0,0.7376213073730469
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,128,256,64,0,0.1544533371925354
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,128,256,32,0,0.2310826579729716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,256,16,4,0,0.3901439905166626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,128,256,2,0,2.7557547887166343
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,128,256,4,0,1.405951976776123
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,256,16,2,0,0.7666347026824951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,256,16,8,0,0.20377600193023682
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,256,16,16,0,0.11212799946467082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,256,16,32,0,0.06502399841944377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,256,16,1,0,1.5228586196899414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,256,16,64,0,0.03908266623814901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,256,16,8,0,0.20428800582885742
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,256,16,16,0,0.11178666353225708
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,256,16,4,0,0.3906559944152832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,256,16,32,0,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,256,16,64,0,0.03908266623814901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,256,16,2,0,0.7635626792907715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,256,16,1,0,1.5199573834737141
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,256,32,8,0,0.233130673567454
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,128,256,2,0,2.7388585408528647
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,256,32,4,0,0.4505600134531657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,256,32,32,0,0.07611733178297679
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,256,32,16,0,0.12748799721399942
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,256,32,64,0,0.05239466826121012
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,256,32,2,0,0.876202662785848
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,256,32,8,0,0.23483733336130777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,256,32,16,0,0.16025599837303162
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,256,32,4,0,0.4751360019048055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,256,32,32,0,0.07645866771539052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,256,32,64,0,0.051370665431022644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,256,32,1,0,1.7317546208699544
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,256,32,2,0,0.8698879877726237
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,256,64,16,0,0.1889280080795288
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,256,64,8,0,0.3415040175120036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,256,64,32,0,0.1129813293615977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,256,64,4,0,0.6586026748021444
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,256,32,1,0,1.7286826769510906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,256,64,64,0,0.08277333279450734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,256,64,8,0,0.34696535269419354
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,128,256,1,0,5.580117543538411
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,256,64,2,0,1.2627627054850261
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,256,64,4,0,0.65774933497111
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,256,64,16,0,0.1930239995320638
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,256,64,32,0,0.11639466881752014
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,256,64,64,0,0.08311466872692108
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,128,256,1,0,5.6079362233479815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,256,64,2,0,1.265664021174113
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,8,256,128,16,0,0.3778560161590576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,4,256,128,32,0,0.222378671169281
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,16,256,128,8,0,0.6891520023345947
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,2,256,128,64,0,0.1397760013739268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,256,64,1,0,2.496682643890381
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,32,256,128,4,0,1.3042346636454265
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,256,64,1,0,2.50163205464681
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,4,256,128,32,0,0.22203733523686728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,8,256,128,16,0,0.3717120091120402
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,16,256,128,8,0,0.6862506866455078
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,2,256,128,64,0,0.14165332913398743
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,32,256,128,4,0,1.3069653511047363
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,64,256,128,2,0,2.571434656778971
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,64,256,128,2,0,2.5501012802124023
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,fp8,128,256,128,1,0,5.084159851074219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_context,default,float16,float16,128,256,128,1,0,5.083135922749837
